(bert_config, is_training, input_ids, input_mask, segment_ids,
                 labels, num_labels, use_one_hot_embeddings)
Creates a classification model.
Creates a classification model.
Creates a classification model .
def create_model(bert_config, is_training, input_ids, input_mask, segment_ids,
                 labels, num_labels, use_one_hot_embeddings):
  """Creates a classification model."""
  model = modeling.BertModel(
      config=bert_config,
      is_training=is_training,
      input_ids=input_ids,
      input_mask=input_mask,
      token_type_ids=segment_ids,
      use_one_hot_embeddings=use_one_hot_embeddings)

  # In the demo, we are doing a simple classification task on the entire
  # segment.
  #
  # If you want to use the token-level output, use model.get_sequence_output()
  # instead.
  output_layer = model.get_pooled_output()

  hidden_size = output_layer.shape[-1].value

  output_weights = tf.get_variable(
      "output_weights", [num_labels, hidden_size],
      initializer=tf.truncated_normal_initializer(stddev=0.02))

  output_bias = tf.get_variable(
      "output_bias", [num_labels], initializer=tf.zeros_initializer())

  with tf.variable_scope("loss"):
    ln_type = bert_config.ln_type
    if ln_type == 'preln':  # add by brightmart, 10-06. if it is preln, we need to an additonal layer: layer normalization as suggested in paper "ON LAYER NORMALIZATION IN THE TRANSFORMER ARCHITECTURE"
      print("ln_type is preln. add LN layer.")
      output_layer = layer_norm(output_layer)
    else:
      print("ln_type is postln or other,do nothing.")

    if is_training:
      # I.e., 0.1 dropout
      output_layer = tf.nn.dropout(output_layer, keep_prob=0.9)

    logits = tf.matmul(output_layer, output_weights, transpose_b=True)
    logits = tf.nn.bias_add(logits, output_bias)
    probabilities = tf.nn.softmax(logits, axis=-1)
    log_probs = tf.nn.log_softmax(logits, axis=-1)

    one_hot_labels = tf.one_hot(labels, depth=num_labels, dtype=tf.float32)

    per_example_loss = -tf.reduce_sum(one_hot_labels * log_probs,
                                      axis=-1)  # todo 08-29 try temp-loss
    ###############bi_tempered_logistic_loss############################################################################
    # print("##cross entropy loss is used...."); tf.logging.info("##cross entropy loss is used....")
    # t1=0.9 #t1=0.90
    # t2=1.05 #t2=1.05
    # per_example_loss=bi_tempered_logistic_loss(log_probs,one_hot_labels,t1,t2,label_smoothing=0.1,num_iters=5) # TODO label_smoothing=0.0
    # tf.logging.info("per_example_loss:"+str(per_example_loss.shape))
    ##############bi_tempered_logistic_loss#############################################################################

    loss = tf.reduce_mean(per_example_loss)

    return (loss, per_example_loss, logits, probabilities)
def create_model ( bert_config , is_training , input_ids , input_mask , segment_ids , labels , num_labels , use_one_hot_embeddings ) : model = modeling . BertModel ( config = bert_config , is_training = is_training , input_ids = input_ids , input_mask = input_mask , token_type_ids = segment_ids , use_one_hot_embeddings = use_one_hot_embeddings ) # In the demo, we are doing a simple classification task on the entire # segment. # # If you want to use the token-level output, use model.get_sequence_output() # instead. output_layer = model . get_pooled_output ( ) hidden_size = output_layer . shape [ - 1 ] . value output_weights = tf . get_variable ( "output_weights" , [ num_labels , hidden_size ] , initializer = tf . truncated_normal_initializer ( stddev = 0.02 ) ) output_bias = tf . get_variable ( "output_bias" , [ num_labels ] , initializer = tf . zeros_initializer ( ) ) with tf . variable_scope ( "loss" ) : ln_type = bert_config . ln_type if ln_type == 'preln' : # add by brightmart, 10-06. if it is preln, we need to an additonal layer: layer normalization as suggested in paper "ON LAYER NORMALIZATION IN THE TRANSFORMER ARCHITECTURE" print ( "ln_type is preln. add LN layer." ) output_layer = layer_norm ( output_layer ) else : print ( "ln_type is postln or other,do nothing." ) if is_training : # I.e., 0.1 dropout output_layer = tf . nn . dropout ( output_layer , keep_prob = 0.9 ) logits = tf . matmul ( output_layer , output_weights , transpose_b = True ) logits = tf . nn . bias_add ( logits , output_bias ) probabilities = tf . nn . softmax ( logits , axis = - 1 ) log_probs = tf . nn . log_softmax ( logits , axis = - 1 ) one_hot_labels = tf . one_hot ( labels , depth = num_labels , dtype = tf . float32 ) per_example_loss = - tf . reduce_sum ( one_hot_labels * log_probs , axis = - 1 ) # todo 08-29 try temp-loss ###############bi_tempered_logistic_loss############################################################################ # print("##cross entropy loss is used...."); tf.logging.info("##cross entropy loss is used....") # t1=0.9 #t1=0.90 # t2=1.05 #t2=1.05 # per_example_loss=bi_tempered_logistic_loss(log_probs,one_hot_labels,t1,t2,label_smoothing=0.1,num_iters=5) # TODO label_smoothing=0.0 # tf.logging.info("per_example_loss:"+str(per_example_loss.shape)) ##############bi_tempered_logistic_loss############################################################################# loss = tf . reduce_mean ( per_example_loss ) return ( loss , per_example_loss , logits , probabilities )