(bert_config, input_tensor, output_weights, positions,
                         label_ids, label_weights)
return (loss, per_example_loss, log_probs)
Get loss and log probs for the masked LM.
Get loss and log probs for the masked LM.
Get loss and log probs for the masked LM .
def get_masked_lm_output(bert_config, input_tensor, output_weights, positions,
                         label_ids, label_weights):
  """Get loss and log probs for the masked LM."""
  input_tensor = gather_indexes(input_tensor, positions)

  with tf.variable_scope("cls/predictions"):
    # We apply one more non-linear transformation before the output layer.
    # This matrix is not used after pre-training.
    with tf.variable_scope("transform"):
      input_tensor = tf.layers.dense(
          input_tensor,
          units=bert_config.hidden_size,
          activation=modeling.get_activation(bert_config.hidden_act),
          kernel_initializer=modeling.create_initializer(
              bert_config.initializer_range))
      input_tensor = modeling.layer_norm(input_tensor)

    # The output weights are the same as the input embeddings, but there is
    # an output-only bias for each token.
    output_bias = tf.get_variable(
        "output_bias",
        shape=[bert_config.vocab_size],
        initializer=tf.zeros_initializer())
    logits = tf.matmul(input_tensor, output_weights, transpose_b=True)
    logits = tf.nn.bias_add(logits, output_bias)
    log_probs = tf.nn.log_softmax(logits, axis=-1)

    label_ids = tf.reshape(label_ids, [-1])
    label_weights = tf.reshape(label_weights, [-1])

    one_hot_labels = tf.one_hot(label_ids, depth=bert_config.vocab_size, dtype=tf.float32)

    # The `positions` tensor might be zero-padded (if the sequence is too
    # short to have the maximum number of predictions). The `label_weights`
    # tensor has a value of 1.0 for every real prediction and 0.0 for the
    # padding predictions.
    per_example_loss = -tf.reduce_sum(log_probs * one_hot_labels, axis=[-1])
    numerator = tf.reduce_sum(label_weights * per_example_loss)
    denominator = tf.reduce_sum(label_weights) + 1e-5
    loss = numerator / denominator

  return (loss, per_example_loss, log_probs)
def get_masked_lm_output ( bert_config , input_tensor , output_weights , positions , label_ids , label_weights ) : input_tensor = gather_indexes ( input_tensor , positions ) with tf . variable_scope ( "cls/predictions" ) : # We apply one more non-linear transformation before the output layer. # This matrix is not used after pre-training. with tf . variable_scope ( "transform" ) : input_tensor = tf . layers . dense ( input_tensor , units = bert_config . hidden_size , activation = modeling . get_activation ( bert_config . hidden_act ) , kernel_initializer = modeling . create_initializer ( bert_config . initializer_range ) ) input_tensor = modeling . layer_norm ( input_tensor ) # The output weights are the same as the input embeddings, but there is # an output-only bias for each token. output_bias = tf . get_variable ( "output_bias" , shape = [ bert_config . vocab_size ] , initializer = tf . zeros_initializer ( ) ) logits = tf . matmul ( input_tensor , output_weights , transpose_b = True ) logits = tf . nn . bias_add ( logits , output_bias ) log_probs = tf . nn . log_softmax ( logits , axis = - 1 ) label_ids = tf . reshape ( label_ids , [ - 1 ] ) label_weights = tf . reshape ( label_weights , [ - 1 ] ) one_hot_labels = tf . one_hot ( label_ids , depth = bert_config . vocab_size , dtype = tf . float32 ) # The `positions` tensor might be zero-padded (if the sequence is too # short to have the maximum number of predictions). The `label_weights` # tensor has a value of 1.0 for every real prediction and 0.0 for the # padding predictions. per_example_loss = - tf . reduce_sum ( log_probs * one_hot_labels , axis = [ - 1 ] ) numerator = tf . reduce_sum ( label_weights * per_example_loss ) denominator = tf . reduce_sum ( label_weights ) + 1e-5 loss = numerator / denominator return ( loss , per_example_loss , log_probs )