(self, layer_list, epoch)
Apply the learning rule to all the layers and update the states.

        Arguments:
            param_list (list): a list of tuples of the form ((param, grad), state),
                               corresponding to parameters, grads,
                               and states of layers to be updated
            epoch (int): the current epoch, needed for the Schedule object.
Apply the learning rule to all the layers and update the states.
Apply the learning rule to all the layers and update the states .
def optimize(self, layer_list, epoch):
        """
        Apply the learning rule to all the layers and update the states.

        Arguments:
            param_list (list): a list of tuples of the form ((param, grad), state),
                               corresponding to parameters, grads,
                               and states of layers to be updated
            epoch (int): the current epoch, needed for the Schedule object.
        """
        epsilon, decay = (self.epsilon, self.decay)

        param_list = get_param_list(layer_list)

        for (param, grad), states in param_list:
            param.rounding = self.stochastic_round

            if len(states) == 0:
                # E[Grad^2], E[Delt^2], updates
                states.extend([self.be.zeros_like(grad) for i in range(3)])

            grad = grad / self.be.bsz
            states[0][:] = states[0] * decay + (1. - decay) * grad * grad
            states[2][:] = self.be.sqrt((states[1] + epsilon) / (states[0] + epsilon)) * grad
            states[1][:] = states[1] * decay + (1. - decay) * states[2] * states[2]

            param[:] = self.clip_value(param - states[2], self.param_clip_value)
def optimize ( self , layer_list , epoch ) : epsilon , decay = ( self . epsilon , self . decay ) param_list = get_param_list ( layer_list ) for ( param , grad ) , states in param_list : param . rounding = self . stochastic_round if len ( states ) == 0 : # E[Grad^2], E[Delt^2], updates states . extend ( [ self . be . zeros_like ( grad ) for i in range ( 3 ) ] ) grad = grad / self . be . bsz states [ 0 ] [ : ] = states [ 0 ] * decay + ( 1. - decay ) * grad * grad states [ 2 ] [ : ] = self . be . sqrt ( ( states [ 1 ] + epsilon ) / ( states [ 0 ] + epsilon ) ) * grad states [ 1 ] [ : ] = states [ 1 ] * decay + ( 1. - decay ) * states [ 2 ] * states [ 2 ] param [ : ] = self . clip_value ( param - states [ 2 ] , self . param_clip_value )