(self, layer_list, epoch)
Apply the learning rule to all the layers and update the states.

        Arguments:
            layer_list (list): a list of Layer objects to optimize.
            epoch (int): the current epoch, needed for the Schedule object.
Apply the learning rule to all the layers and update the states.
Apply the learning rule to all the layers and update the states .
def optimize(self, layer_list, epoch):
        """
        Apply the learning rule to all the layers and update the states.

        Arguments:
            layer_list (list): a list of Layer objects to optimize.
            epoch (int): the current epoch, needed for the Schedule object.
        """
        lrate = self.schedule.get_learning_rate(self.learning_rate, epoch)
        param_list = get_param_list(layer_list)

        scale_factor = self.clip_gradient_norm(param_list, self.gradient_clip_norm)

        for (param, grad), states in param_list:
            param.rounding = self.stochastic_round
            if len(states) == 0 and self.momentum_coef != 0:
                states.append(self.be.zeros_like(grad))

            grad = grad / self.be.bsz
            grad = self.clip_value(grad, self.gradient_clip_value)

            if self.momentum_coef == 0:
                param[:] = (- lrate * scale_factor) * grad +\
                           (1 - lrate * self.wdecay) * param
                param = self.clip_value(param, self.param_clip_value)
            else:
                grad = scale_factor * grad + self.wdecay * param
                velocity = states[0]
                velocity[:] = self.momentum_coef * velocity - lrate * grad

                # Nesterov accelerated gradient (NAG) is implemented the same
                # as in torch's "sgd.lua". It's a reformulation of Sutskever's
                # NAG equation found in "On the importance of initialization
                # and momentum in deep learning".
                if self.nesterov:
                    param[:] = self.clip_value(
                               param + self.momentum_coef * velocity
                               - lrate * grad, self.param_clip_value)
                else:
                    param[:] = self.clip_value(
                                param + velocity, self.param_clip_value)
def optimize ( self , layer_list , epoch ) : lrate = self . schedule . get_learning_rate ( self . learning_rate , epoch ) param_list = get_param_list ( layer_list ) scale_factor = self . clip_gradient_norm ( param_list , self . gradient_clip_norm ) for ( param , grad ) , states in param_list : param . rounding = self . stochastic_round if len ( states ) == 0 and self . momentum_coef != 0 : states . append ( self . be . zeros_like ( grad ) ) grad = grad / self . be . bsz grad = self . clip_value ( grad , self . gradient_clip_value ) if self . momentum_coef == 0 : param [ : ] = ( - lrate * scale_factor ) * grad + ( 1 - lrate * self . wdecay ) * param param = self . clip_value ( param , self . param_clip_value ) else : grad = scale_factor * grad + self . wdecay * param velocity = states [ 0 ] velocity [ : ] = self . momentum_coef * velocity - lrate * grad # Nesterov accelerated gradient (NAG) is implemented the same # as in torch's "sgd.lua". It's a reformulation of Sutskever's # NAG equation found in "On the importance of initialization # and momentum in deep learning". if self . nesterov : param [ : ] = self . clip_value ( param + self . momentum_coef * velocity - lrate * grad , self . param_clip_value ) else : param [ : ] = self . clip_value ( param + velocity , self . param_clip_value )