(self, layer_list, epoch)
Apply the learning rule to all the layers and update the states.

        Arguments:
            layer_list (list): a list of Layer objects to optimize.
            epoch (int): the current epoch, needed for the Schedule object.
Apply the learning rule to all the layers and update the states.
Apply the learning rule to all the layers and update the states .
def optimize(self, layer_list, epoch):
        """
        Apply the learning rule to all the layers and update the states.

        Arguments:
            layer_list (list): a list of Layer objects to optimize.
            epoch (int): the current epoch, needed for the Schedule object.
        """
        lrate = self.schedule.get_learning_rate(self.learning_rate, epoch)
        epsilon, decay = (self.epsilon, self.decay_rate)
        param_list = get_param_list(layer_list)

        scale_factor = self.clip_gradient_norm(param_list, self.gradient_clip_norm)

        for (param, grad), states in param_list:

            param.rounding = self.stochastic_round
            if len(states) == 0:
                states.append(self.be.zeros_like(grad))

            grad = grad / self.be.bsz
            grad = self.clip_value(grad, self.gradient_clip_value)

            # update state
            state = states[0]
            state[:] = decay * state + self.be.square(grad) * (1.0 - decay)

            param[:] = self.clip_value(
                        param - (scale_factor * grad * lrate)
                        / (self.be.sqrt(state + epsilon) + epsilon),
                        self.param_clip_value)
def optimize ( self , layer_list , epoch ) : lrate = self . schedule . get_learning_rate ( self . learning_rate , epoch ) epsilon , decay = ( self . epsilon , self . decay_rate ) param_list = get_param_list ( layer_list ) scale_factor = self . clip_gradient_norm ( param_list , self . gradient_clip_norm ) for ( param , grad ) , states in param_list : param . rounding = self . stochastic_round if len ( states ) == 0 : states . append ( self . be . zeros_like ( grad ) ) grad = grad / self . be . bsz grad = self . clip_value ( grad , self . gradient_clip_value ) # update state state = states [ 0 ] state [ : ] = decay * state + self . be . square ( grad ) * ( 1.0 - decay ) param [ : ] = self . clip_value ( param - ( scale_factor * grad * lrate ) / ( self . be . sqrt ( state + epsilon ) + epsilon ) , self . param_clip_value )