(hidden, target, n_token, d_model, initializer, lookup_table=None,
            tie_weight=False, bi_data=True, use_tpu=False)
doc.
doc.
doc .
def lm_loss(hidden, target, n_token, d_model, initializer, lookup_table=None,
            tie_weight=False, bi_data=True, use_tpu=False):
  """doc."""

  with tf.variable_scope('lm_loss'):
    if tie_weight:
      assert lookup_table is not None, \
          'lookup_table cannot be None for tie_weight'
      softmax_w = lookup_table
    else:
      softmax_w = tf.get_variable('weight', [n_token, d_model],
                                  dtype=hidden.dtype, initializer=initializer)

    softmax_b = tf.get_variable('bias', [n_token], dtype=hidden.dtype,
                                initializer=tf.zeros_initializer())

    logits = tf.einsum('ibd,nd->ibn', hidden, softmax_w) + softmax_b

    if use_tpu:
      one_hot_target = tf.one_hot(target, n_token, dtype=logits.dtype)
      loss = -tf.reduce_sum(tf.nn.log_softmax(logits) * one_hot_target, -1)
    else:
      loss = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=target,
                                                            logits=logits)

    return loss
def lm_loss ( hidden , target , n_token , d_model , initializer , lookup_table = None , tie_weight = False , bi_data = True , use_tpu = False ) : with tf . variable_scope ( 'lm_loss' ) : if tie_weight : assert lookup_table is not None , 'lookup_table cannot be None for tie_weight' softmax_w = lookup_table else : softmax_w = tf . get_variable ( 'weight' , [ n_token , d_model ] , dtype = hidden . dtype , initializer = initializer ) softmax_b = tf . get_variable ( 'bias' , [ n_token ] , dtype = hidden . dtype , initializer = tf . zeros_initializer ( ) ) logits = tf . einsum ( 'ibd,nd->ibn' , hidden , softmax_w ) + softmax_b if use_tpu : one_hot_target = tf . one_hot ( target , n_token , dtype = logits . dtype ) loss = - tf . reduce_sum ( tf . nn . log_softmax ( logits ) * one_hot_target , - 1 ) else : loss = tf . nn . sparse_softmax_cross_entropy_with_logits ( labels = target , logits = logits ) return loss