(self, x)
return tf.nn.batch_normalization(x, mean, variance, betas, gammas,
                                     self.hparams.batch_norm_variance_epsilon)
Normalizes batch w/ moving population stats for training, o/w batch.
Normalizes batch w/ moving population stats for training, o/w batch.
Normalizes batch w / moving population stats for training o / w batch .
def apply_batchnorm(self, x):
    """Normalizes batch w/ moving population stats for training, o/w batch."""
    output_dim = x.get_shape()[-1]
    gammas = tf.get_variable(
        'gamma', [1, 1, 1, output_dim],
        initializer=tf.constant_initializer(0.1))
    betas = tf.get_variable(
        'beta', [output_dim], initializer=tf.constant_initializer(0.))

    popmean = tf.get_variable(
        'popmean',
        shape=[1, 1, 1, output_dim],
        trainable=False,
        collections=[
            tf.GraphKeys.MODEL_VARIABLES, tf.GraphKeys.GLOBAL_VARIABLES
        ],
        initializer=tf.constant_initializer(0.0))
    popvariance = tf.get_variable(
        'popvariance',
        shape=[1, 1, 1, output_dim],
        trainable=False,
        collections=[
            tf.GraphKeys.MODEL_VARIABLES, tf.GraphKeys.GLOBAL_VARIABLES
        ],
        initializer=tf.constant_initializer(1.0))

    decay = 0.01
    if self.is_training:
      batchmean, batchvariance = tf.nn.moments(x, [0, 1, 2], keep_dims=True)
      mean, variance = batchmean, batchvariance
      updates = [
          popmean.assign_sub(decay * (popmean - mean)),
          popvariance.assign_sub(decay * (popvariance - variance))
      ]
      # make update happen when mean/variance are used
      with tf.control_dependencies(updates):
        mean, variance = tf.identity(mean), tf.identity(variance)
      self.popstats_by_batchstat[batchmean] = popmean
      self.popstats_by_batchstat[batchvariance] = popvariance
    else:
      mean, variance = popmean, popvariance

    return tf.nn.batch_normalization(x, mean, variance, betas, gammas,
                                     self.hparams.batch_norm_variance_epsilon)
def apply_batchnorm ( self , x ) : output_dim = x . get_shape ( ) [ - 1 ] gammas = tf . get_variable ( 'gamma' , [ 1 , 1 , 1 , output_dim ] , initializer = tf . constant_initializer ( 0.1 ) ) betas = tf . get_variable ( 'beta' , [ output_dim ] , initializer = tf . constant_initializer ( 0. ) ) popmean = tf . get_variable ( 'popmean' , shape = [ 1 , 1 , 1 , output_dim ] , trainable = False , collections = [ tf . GraphKeys . MODEL_VARIABLES , tf . GraphKeys . GLOBAL_VARIABLES ] , initializer = tf . constant_initializer ( 0.0 ) ) popvariance = tf . get_variable ( 'popvariance' , shape = [ 1 , 1 , 1 , output_dim ] , trainable = False , collections = [ tf . GraphKeys . MODEL_VARIABLES , tf . GraphKeys . GLOBAL_VARIABLES ] , initializer = tf . constant_initializer ( 1.0 ) ) decay = 0.01 if self . is_training : batchmean , batchvariance = tf . nn . moments ( x , [ 0 , 1 , 2 ] , keep_dims = True ) mean , variance = batchmean , batchvariance updates = [ popmean . assign_sub ( decay * ( popmean - mean ) ) , popvariance . assign_sub ( decay * ( popvariance - variance ) ) ] # make update happen when mean/variance are used with tf . control_dependencies ( updates ) : mean , variance = tf . identity ( mean ) , tf . identity ( variance ) self . popstats_by_batchstat [ batchmean ] = popmean self . popstats_by_batchstat [ batchvariance ] = popvariance else : mean , variance = popmean , popvariance return tf . nn . batch_normalization ( x , mean , variance , betas , gammas , self . hparams . batch_norm_variance_epsilon )