(x,
               num_units,
               scope='layer_norm',
               reuse=False,
               gamma_start=1.0,
               epsilon=1e-3,
               use_bias=True)
return output
Calculate layer norm.
Calculate layer norm.
Calculate layer norm .
def layer_norm(x,
               num_units,
               scope='layer_norm',
               reuse=False,
               gamma_start=1.0,
               epsilon=1e-3,
               use_bias=True):
  """Calculate layer norm."""
  axes = [1]
  mean = tf.reduce_mean(x, axes, keep_dims=True)
  x_shifted = x - mean
  var = tf.reduce_mean(tf.square(x_shifted), axes, keep_dims=True)
  inv_std = tf.rsqrt(var + epsilon)
  with tf.variable_scope(scope):
    if reuse:
      tf.get_variable_scope().reuse_variables()
    gamma = tf.get_variable(
        'ln_gamma', [num_units],
        initializer=tf.constant_initializer(gamma_start))
    if use_bias:
      beta = tf.get_variable(
          'ln_beta', [num_units], initializer=tf.constant_initializer(0.0))
  output = gamma * (x_shifted) * inv_std
  if use_bias:
    output += beta
  return output
def layer_norm ( x , num_units , scope = 'layer_norm' , reuse = False , gamma_start = 1.0 , epsilon = 1e-3 , use_bias = True ) : axes = [ 1 ] mean = tf . reduce_mean ( x , axes , keep_dims = True ) x_shifted = x - mean var = tf . reduce_mean ( tf . square ( x_shifted ) , axes , keep_dims = True ) inv_std = tf . rsqrt ( var + epsilon ) with tf . variable_scope ( scope ) : if reuse : tf . get_variable_scope ( ) . reuse_variables ( ) gamma = tf . get_variable ( 'ln_gamma' , [ num_units ] , initializer = tf . constant_initializer ( gamma_start ) ) if use_bias : beta = tf . get_variable ( 'ln_beta' , [ num_units ] , initializer = tf . constant_initializer ( 0.0 ) ) output = gamma * ( x_shifted ) * inv_std if use_bias : output += beta return output