(self,
               learning_rate=0.0001,
               momentum=0.0,
               clip_thresh=None,
               beta=0.999,
               curv_win_width=20,
               zero_debias=True,
               delta_mu=0.0,
               sparsity_debias=False,
               use_locking=False,
               name="YellowFin",
               use_nesterov=False,
               use_unsmoothed_lr_mu=True,
               h_max_log_smooth=True,
               h_min_log_smooth=True,
               use_adapt_grad_clip=True,
               stat_protect_fac=100.0)
Construct a new YellowFin optimizer.

    Args:
      learning rate: Python scalar. The initial value of learning rate,
        we use 1.0 in our paper.
      momentum: Python scalar. The initial value of momentum, we use
        0.0 in our paper.
      clip_thresh: Python scalar. The cliping threshold for
        `tf.clip_by_global_norm`. If None, no clipping will be used.
      beta: Python scalar. The smoothing parameter for estimations.
      curv_win_width: TODO
      zero_debias: TODO
      delta_mu: for extensions. Not necessary in the basic use.
      sparsity_debias: Python boolean. Gradient norm and curvature are
        biased to larger values when calculated with sparse gradient.
        This is useful when the model is very sparse, e.g. LSTM with
        word embedding. For non-sparse CNN, turning it off could
        slightly accelerate the speed.
      use_locking: If True, use locks for update operations.
      name: Optional name prefix for the operations created when
        applying gradients. Defaults to "YellowFin".
      use_nesterov: If True, the underlying MomentumOptimizer uses Nesterov
        Momentum. Set to False in the default YellowFin algorithm.

    Notes:
      `clip_thresh` is the threshold value on ||lr * gradient||
      `delta_mu` can be a placeholder/variable/python scalar. Used for
      additional momentum in situations such as asynchronous-parallel
      training. The default is 0.0 for basic usage of the optimizer.

    Other features:
      If you want to manually control the learning rates,
      `self.lr_factor` is an interface to the outside. It is a
      multiplier for the internal learning rate in YellowFin. It is
      helpful when you want to do additional hand tuning or some
      decaying scheme for the internal learning rate. Example on using
      `lr_factor` can be found here:
      https://github.com/JianGoForIt/YellowFin/blob/master/char-rnn-tensorflow/train_YF.py#L140
Construct a new YellowFin optimizer.
Construct a new YellowFin optimizer .
def __init__(self,
               learning_rate=0.0001,
               momentum=0.0,
               clip_thresh=None,
               beta=0.999,
               curv_win_width=20,
               zero_debias=True,
               delta_mu=0.0,
               sparsity_debias=False,
               use_locking=False,
               name="YellowFin",
               use_nesterov=False,
               use_unsmoothed_lr_mu=True,
               h_max_log_smooth=True,
               h_min_log_smooth=True,
               use_adapt_grad_clip=True,
               stat_protect_fac=100.0):
    """
    Construct a new YellowFin optimizer.

    Args:
      learning rate: Python scalar. The initial value of learning rate,
        we use 1.0 in our paper.
      momentum: Python scalar. The initial value of momentum, we use
        0.0 in our paper.
      clip_thresh: Python scalar. The cliping threshold for
        `tf.clip_by_global_norm`. If None, no clipping will be used.
      beta: Python scalar. The smoothing parameter for estimations.
      curv_win_width: TODO
      zero_debias: TODO
      delta_mu: for extensions. Not necessary in the basic use.
      sparsity_debias: Python boolean. Gradient norm and curvature are
        biased to larger values when calculated with sparse gradient.
        This is useful when the model is very sparse, e.g. LSTM with
        word embedding. For non-sparse CNN, turning it off could
        slightly accelerate the speed.
      use_locking: If True, use locks for update operations.
      name: Optional name prefix for the operations created when
        applying gradients. Defaults to "YellowFin".
      use_nesterov: If True, the underlying MomentumOptimizer uses Nesterov
        Momentum. Set to False in the default YellowFin algorithm.

    Notes:
      `clip_thresh` is the threshold value on ||lr * gradient||
      `delta_mu` can be a placeholder/variable/python scalar. Used for
      additional momentum in situations such as asynchronous-parallel
      training. The default is 0.0 for basic usage of the optimizer.

    Other features:
      If you want to manually control the learning rates,
      `self.lr_factor` is an interface to the outside. It is a
      multiplier for the internal learning rate in YellowFin. It is
      helpful when you want to do additional hand tuning or some
      decaying scheme for the internal learning rate. Example on using
      `lr_factor` can be found here:
      https://github.com/JianGoForIt/YellowFin/blob/master/char-rnn-tensorflow/train_YF.py#L140
    """
    self._lr = learning_rate
    self._mu = momentum

    self._lr_var = tf.Variable(
        learning_rate, dtype=tf.float32, name="YF_lr", trainable=False)
    self._mu_var = tf.Variable(
        momentum, dtype=tf.float32, name="YF_mu", trainable=False)
    # for step scheme or decaying scheme for the learning rates
    self.lr_factor = tf.Variable(
        1.0, dtype=tf.float32, name="YF_lr_factor", trainable=False)
    if clip_thresh is not None:
      self._clip_thresh_var = tf.Variable(
          clip_thresh, dtype=tf.float32, name="YF_clip_thresh", trainable=False)
    else:
      self._clip_thresh_var = None

    # the underlying momentum optimizer
    self._optimizer = tf.train.MomentumOptimizer(self._lr_var * self.lr_factor,
                                                 self._mu_var + delta_mu,
                                                 use_locking, name,
                                                 use_nesterov)

    # moving average for statistics
    self._beta = beta
    self._moving_averager = None

    # for global step counting
    self._global_step = tf.Variable(0, trainable=False)

    self._do_tune = tf.greater(self._global_step, tf.constant(0))

    self._zero_debias = zero_debias
    self._sparsity_debias = sparsity_debias

    self._tvars = None

    # for curvature range
    self._curv_win_width = curv_win_width
    self._curv_win = None

    # option for using smoothed or unsmoothed lr and mu
    self._use_unsmoothed_lr_mu = use_unsmoothed_lr_mu

    # options for curvature envelop smoothing
    self._h_max_log_smooth = h_max_log_smooth
    self._h_min_log_smooth = h_min_log_smooth

    # for adaptive gradient clipping
    self._use_adapt_grad_clip = use_adapt_grad_clip
    self._adapt_grad_clip_thresh = \
      tf.Variable(LARGE_FLOAT_VAL, dtype=tf.float32, trainable=False)
    self._adapt_grad_clip_target_val = \
      tf.Variable(LARGE_FLOAT_VAL, dtype=tf.float32, trainable=False)

    # prevent exploding gradient from ruining the statistics
    self._stat_protect_fac = stat_protect_fac
def __init__ ( self , learning_rate = 0.0001 , momentum = 0.0 , clip_thresh = None , beta = 0.999 , curv_win_width = 20 , zero_debias = True , delta_mu = 0.0 , sparsity_debias = False , use_locking = False , name = "YellowFin" , use_nesterov = False , use_unsmoothed_lr_mu = True , h_max_log_smooth = True , h_min_log_smooth = True , use_adapt_grad_clip = True , stat_protect_fac = 100.0 ) : self . _lr = learning_rate self . _mu = momentum self . _lr_var = tf . Variable ( learning_rate , dtype = tf . float32 , name = "YF_lr" , trainable = False ) self . _mu_var = tf . Variable ( momentum , dtype = tf . float32 , name = "YF_mu" , trainable = False ) # for step scheme or decaying scheme for the learning rates self . lr_factor = tf . Variable ( 1.0 , dtype = tf . float32 , name = "YF_lr_factor" , trainable = False ) if clip_thresh is not None : self . _clip_thresh_var = tf . Variable ( clip_thresh , dtype = tf . float32 , name = "YF_clip_thresh" , trainable = False ) else : self . _clip_thresh_var = None # the underlying momentum optimizer self . _optimizer = tf . train . MomentumOptimizer ( self . _lr_var * self . lr_factor , self . _mu_var + delta_mu , use_locking , name , use_nesterov ) # moving average for statistics self . _beta = beta self . _moving_averager = None # for global step counting self . _global_step = tf . Variable ( 0 , trainable = False ) self . _do_tune = tf . greater ( self . _global_step , tf . constant ( 0 ) ) self . _zero_debias = zero_debias self . _sparsity_debias = sparsity_debias self . _tvars = None # for curvature range self . _curv_win_width = curv_win_width self . _curv_win = None # option for using smoothed or unsmoothed lr and mu self . _use_unsmoothed_lr_mu = use_unsmoothed_lr_mu # options for curvature envelop smoothing self . _h_max_log_smooth = h_max_log_smooth self . _h_min_log_smooth = h_min_log_smooth # for adaptive gradient clipping self . _use_adapt_grad_clip = use_adapt_grad_clip self . _adapt_grad_clip_thresh = tf . Variable ( LARGE_FLOAT_VAL , dtype = tf . float32 , trainable = False ) self . _adapt_grad_clip_target_val = tf . Variable ( LARGE_FLOAT_VAL , dtype = tf . float32 , trainable = False ) # prevent exploding gradient from ruining the statistics self . _stat_protect_fac = stat_protect_fac