(self, layer_list, epoch)
Apply the learning rule to all the layers and update the states.

        Arguments:
            param_list (list): a list of tuples of the form ((param, grad), state),
                corresponding to parameters, grads, and states of layers to be updated
            epoch (int): the current epoch, needed for the Schedule object.
Apply the learning rule to all the layers and update the states.
Apply the learning rule to all the layers and update the states .
def optimize(self, layer_list, epoch):
        """
        Apply the learning rule to all the layers and update the states.

        Arguments:
            param_list (list): a list of tuples of the form ((param, grad), state),
                corresponding to parameters, grads, and states of layers to be updated
            epoch (int): the current epoch, needed for the Schedule object.
        """
        t = epoch + 1
        lrate = self.schedule.get_learning_rate(self.learning_rate, epoch)
        l = lrate / (1 - self.beta_1 ** t)

        param_list = get_param_list(layer_list)

        for (param, grad), states in param_list:
            param.rounding = self.stochastic_round
            if len(states) == 0:
                # running_1st_mom, running_2nd_mom
                states.extend([self.be.zeros_like(grad) for i in range(3)])

            grad = grad / self.be.bsz
            m, v, inv_v = states
            m[:] = m * self.beta_1 + (1. - self.beta_1) * grad
            v[:] = self.be.maximum(v * self.beta_2, self.be.absolute(grad))

            inv_v[:] = 1.0 / (v + self.epsilon)
            param[:] = param - self.be.shift(self.be.shift(m, inv_v), l)
            self.be.clip(param, -1, 1, param)
def optimize ( self , layer_list , epoch ) : t = epoch + 1 lrate = self . schedule . get_learning_rate ( self . learning_rate , epoch ) l = lrate / ( 1 - self . beta_1 ** t ) param_list = get_param_list ( layer_list ) for ( param , grad ) , states in param_list : param . rounding = self . stochastic_round if len ( states ) == 0 : # running_1st_mom, running_2nd_mom states . extend ( [ self . be . zeros_like ( grad ) for i in range ( 3 ) ] ) grad = grad / self . be . bsz m , v , inv_v = states m [ : ] = m * self . beta_1 + ( 1. - self . beta_1 ) * grad v [ : ] = self . be . maximum ( v * self . beta_2 , self . be . absolute ( grad ) ) inv_v [ : ] = 1.0 / ( v + self . epsilon ) param [ : ] = param - self . be . shift ( self . be . shift ( m , inv_v ) , l ) self . be . clip ( param , - 1 , 1 , param )