(inputs, axis, parallel=True, group_size=None)
return mean, variance
Compute mean and variance of the inputs tensor across TPU replicas.

  Args:
    inputs: A tensor with 2 or more dimensions.
    axis: Array of ints. Axes along which to compute mean and variance.
    parallel: Use E[x^2] - (E[x])^2 to compute variance. Then can be done
      in parallel to computing the mean and reducing the communication overhead.
    group_size: Integer, the number of replicas to compute moments arcoss.
      None or 0 will use all replicas (global).

  Returns:
    Two tensors with mean and variance.
Compute mean and variance of the inputs tensor across TPU replicas.
Compute mean and variance of the inputs tensor across TPU replicas .
def cross_replica_moments(inputs, axis, parallel=True, group_size=None):
  """Compute mean and variance of the inputs tensor across TPU replicas.

  Args:
    inputs: A tensor with 2 or more dimensions.
    axis: Array of ints. Axes along which to compute mean and variance.
    parallel: Use E[x^2] - (E[x])^2 to compute variance. Then can be done
      in parallel to computing the mean and reducing the communication overhead.
    group_size: Integer, the number of replicas to compute moments arcoss.
      None or 0 will use all replicas (global).

  Returns:
    Two tensors with mean and variance.
  """
  # Compute local mean and then average across replicas.
  mean = tf.math.reduce_mean(inputs, axis=axis)
  mean = cross_replica_mean(mean)
  if parallel:
    # Compute variance using the E[x^2] - (E[x])^2 formula. This is less
    # numerically stable than the E[(x-E[x])^2] formula, but allows the two
    # cross-replica sums to be computed in parallel, saving communication
    # overhead.
    mean_of_squares = tf.reduce_mean(tf.square(inputs), axis=axis)
    mean_of_squares = cross_replica_mean(mean_of_squares, group_size=group_size)
    mean_squared = tf.square(mean)
    variance = mean_of_squares - mean_squared
  else:
    variance = tf.math.reduce_mean(
        tf.math.square(inputs - mean), axis=axis)
  variance = cross_replica_mean(variance, group_size=group_size)
  return mean, variance
def cross_replica_moments ( inputs , axis , parallel = True , group_size = None ) : # Compute local mean and then average across replicas. mean = tf . math . reduce_mean ( inputs , axis = axis ) mean = cross_replica_mean ( mean ) if parallel : # Compute variance using the E[x^2] - (E[x])^2 formula. This is less # numerically stable than the E[(x-E[x])^2] formula, but allows the two # cross-replica sums to be computed in parallel, saving communication # overhead. mean_of_squares = tf . reduce_mean ( tf . square ( inputs ) , axis = axis ) mean_of_squares = cross_replica_mean ( mean_of_squares , group_size = group_size ) mean_squared = tf . square ( mean ) variance = mean_of_squares - mean_squared else : variance = tf . math . reduce_mean ( tf . math . square ( inputs - mean ) , axis = axis ) variance = cross_replica_mean ( variance , group_size = group_size ) return mean , variance