(self, g, m, beta1)
return m, update
Handle various types of momentum.
Handle various types of momentum.
Handle various types of momentum .
def _momentum_update(self, g, m, beta1):
    """Handle various types of momentum."""
    if self._momentum_type == MomentumType.EMA:
      m = (1 - beta1) * g + beta1 * m
      update = m
    elif self._momentum_type == MomentumType.HEAVY_BALL:
      m = g + beta1 * m
      update = m
    elif self._momentum_type == MomentumType.NESTEROV:
      m = g + beta1 * m
      nesterov_m = g + beta1 * m
      update = nesterov_m
    else:
      assert False, 'Unknown momentum_type.'
    return m, update
def _momentum_update ( self , g , m , beta1 ) : if self . _momentum_type == MomentumType . EMA : m = ( 1 - beta1 ) * g + beta1 * m update = m elif self . _momentum_type == MomentumType . HEAVY_BALL : m = g + beta1 * m update = m elif self . _momentum_type == MomentumType . NESTEROV : m = g + beta1 * m nesterov_m = g + beta1 * m update = nesterov_m else : assert False , 'Unknown momentum_type.' return m , update