(input_, output_size,
                       init=False, init_scale=1.0,
                       name="wn_linear",
                       initializer=tf.truncated_normal_initializer,
                       stddev=0.02)
Linear layer with Weight Normalization (Salimans, Kingma '16).
Linear layer with Weight Normalization (Salimans, Kingma '16).
Linear layer with Weight Normalization ( Salimans Kingma 16 ) .
def weight_norm_linear(input_, output_size,
                       init=False, init_scale=1.0,
                       name="wn_linear",
                       initializer=tf.truncated_normal_initializer,
                       stddev=0.02):
  """Linear layer with Weight Normalization (Salimans, Kingma '16)."""
  with tf.variable_scope(name):
    if init:
      v = tf.get_variable("V", [int(input_.get_shape()[1]), output_size],
                          tf.float32, initializer(0, stddev), trainable=True)
      v_norm = tf.nn.l2_normalize(v.initialized_value(), [0])
      x_init = tf.matmul(input_, v_norm)
      m_init, v_init = tf.nn.moments(x_init, [0])
      scale_init = init_scale / tf.sqrt(v_init + 1e-10)
      g = tf.get_variable("g", dtype=tf.float32,
                          initializer=scale_init, trainable=True)
      b = tf.get_variable("b", dtype=tf.float32, initializer=
                          -m_init*scale_init, trainable=True)
      x_init = tf.reshape(scale_init, [1, output_size]) * (
          x_init - tf.reshape(m_init, [1, output_size]))
      return x_init
    else:
      # Note that the original implementation uses Polyak averaging.
      v = tf.get_variable("V")
      g = tf.get_variable("g")
      b = tf.get_variable("b")
      tf.assert_variables_initialized([v, g, b])
      x = tf.matmul(input_, v)
      scaler = g / tf.sqrt(tf.reduce_sum(tf.square(v), [0]))
      x = tf.reshape(scaler, [1, output_size]) * x + tf.reshape(
          b, [1, output_size])
      return x
def weight_norm_linear ( input_ , output_size , init = False , init_scale = 1.0 , name = "wn_linear" , initializer = tf . truncated_normal_initializer , stddev = 0.02 ) : with tf . variable_scope ( name ) : if init : v = tf . get_variable ( "V" , [ int ( input_ . get_shape ( ) [ 1 ] ) , output_size ] , tf . float32 , initializer ( 0 , stddev ) , trainable = True ) v_norm = tf . nn . l2_normalize ( v . initialized_value ( ) , [ 0 ] ) x_init = tf . matmul ( input_ , v_norm ) m_init , v_init = tf . nn . moments ( x_init , [ 0 ] ) scale_init = init_scale / tf . sqrt ( v_init + 1e-10 ) g = tf . get_variable ( "g" , dtype = tf . float32 , initializer = scale_init , trainable = True ) b = tf . get_variable ( "b" , dtype = tf . float32 , initializer = - m_init * scale_init , trainable = True ) x_init = tf . reshape ( scale_init , [ 1 , output_size ] ) * ( x_init - tf . reshape ( m_init , [ 1 , output_size ] ) ) return x_init else : # Note that the original implementation uses Polyak averaging. v = tf . get_variable ( "V" ) g = tf . get_variable ( "g" ) b = tf . get_variable ( "b" ) tf . assert_variables_initialized ( [ v , g , b ] ) x = tf . matmul ( input_ , v ) scaler = g / tf . sqrt ( tf . reduce_sum ( tf . square ( v ) , [ 0 ] ) ) x = tf . reshape ( scaler , [ 1 , output_size ] ) * x + tf . reshape ( b , [ 1 , output_size ] ) return x