(model, optimizer_cfg)
Build optimizer from configs.

    Args:
        model (:obj:`nn.Module`): The model with parameters to be optimized.
        optimizer_cfg (dict): The config dict of the optimizer.
            Positional fields are:
                - type: class name of the optimizer.
                - lr: base learning rate.
            Optional fields are:
                - any arguments of the corresponding optimizer type, e.g.,
                  weight_decay, momentum, etc.
                - paramwise_options: a dict with 3 accepted fileds
                  (bias_lr_mult, bias_decay_mult, norm_decay_mult).
                  `bias_lr_mult` and `bias_decay_mult` will be multiplied to
                  the lr and weight decay respectively for all bias parameters
                  (except for the normalization layers), and
                  `norm_decay_mult` will be multiplied to the weight decay
                  for all weight and bias parameters of normalization layers.

    Returns:
        torch.optim.Optimizer: The initialized optimizer.
Build optimizer from configs.
Build optimizer from configs .
def build_optimizer(model, optimizer_cfg):
    """Build optimizer from configs.

    Args:
        model (:obj:`nn.Module`): The model with parameters to be optimized.
        optimizer_cfg (dict): The config dict of the optimizer.
            Positional fields are:
                - type: class name of the optimizer.
                - lr: base learning rate.
            Optional fields are:
                - any arguments of the corresponding optimizer type, e.g.,
                  weight_decay, momentum, etc.
                - paramwise_options: a dict with 3 accepted fileds
                  (bias_lr_mult, bias_decay_mult, norm_decay_mult).
                  `bias_lr_mult` and `bias_decay_mult` will be multiplied to
                  the lr and weight decay respectively for all bias parameters
                  (except for the normalization layers), and
                  `norm_decay_mult` will be multiplied to the weight decay
                  for all weight and bias parameters of normalization layers.

    Returns:
        torch.optim.Optimizer: The initialized optimizer.
    """
    if hasattr(model, 'module'):
        model = model.module

    optimizer_cfg = optimizer_cfg.copy()
    paramwise_options = optimizer_cfg.pop('paramwise_options', None)
    # if no paramwise option is specified, just use the global setting
    if paramwise_options is None:
        return obj_from_dict(optimizer_cfg, torch.optim,
                             dict(params=model.parameters()))
    else:
        assert isinstance(paramwise_options, dict)
        # get base lr and weight decay
        base_lr = optimizer_cfg['lr']
        base_wd = optimizer_cfg.get('weight_decay', None)
        # weight_decay must be explicitly specified if mult is specified
        if ('bias_decay_mult' in paramwise_options
                or 'norm_decay_mult' in paramwise_options):
            assert base_wd is not None
        # get param-wise options
        bias_lr_mult = paramwise_options.get('bias_lr_mult', 1.)
        bias_decay_mult = paramwise_options.get('bias_decay_mult', 1.)
        norm_decay_mult = paramwise_options.get('norm_decay_mult', 1.)
        # set param-wise lr and weight decay
        params = []
        for name, param in model.named_parameters():
            param_group = {'params': [param]}
            if not param.requires_grad:
                # FP16 training needs to copy gradient/weight between master
                # weight copy and model weight, it is convenient to keep all
                # parameters here to align with model.parameters()
                params.append(param_group)
                continue

            # for norm layers, overwrite the weight decay of weight and bias
            # TODO: obtain the norm layer prefixes dynamically
            if re.search(r'(bn|gn)(\d+)?.(weight|bias)', name):
                if base_wd is not None:
                    param_group['weight_decay'] = base_wd * norm_decay_mult
            # for other layers, overwrite both lr and weight decay of bias
            elif name.endswith('.bias'):
                param_group['lr'] = base_lr * bias_lr_mult
                if base_wd is not None:
                    param_group['weight_decay'] = base_wd * bias_decay_mult
            # otherwise use the global settings

            params.append(param_group)

        optimizer_cls = getattr(torch.optim, optimizer_cfg.pop('type'))
        return optimizer_cls(params, **optimizer_cfg)
def build_optimizer ( model , optimizer_cfg ) : if hasattr ( model , 'module' ) : model = model . module optimizer_cfg = optimizer_cfg . copy ( ) paramwise_options = optimizer_cfg . pop ( 'paramwise_options' , None ) # if no paramwise option is specified, just use the global setting if paramwise_options is None : return obj_from_dict ( optimizer_cfg , torch . optim , dict ( params = model . parameters ( ) ) ) else : assert isinstance ( paramwise_options , dict ) # get base lr and weight decay base_lr = optimizer_cfg [ 'lr' ] base_wd = optimizer_cfg . get ( 'weight_decay' , None ) # weight_decay must be explicitly specified if mult is specified if ( 'bias_decay_mult' in paramwise_options or 'norm_decay_mult' in paramwise_options ) : assert base_wd is not None # get param-wise options bias_lr_mult = paramwise_options . get ( 'bias_lr_mult' , 1. ) bias_decay_mult = paramwise_options . get ( 'bias_decay_mult' , 1. ) norm_decay_mult = paramwise_options . get ( 'norm_decay_mult' , 1. ) # set param-wise lr and weight decay params = [ ] for name , param in model . named_parameters ( ) : param_group = { 'params' : [ param ] } if not param . requires_grad : # FP16 training needs to copy gradient/weight between master # weight copy and model weight, it is convenient to keep all # parameters here to align with model.parameters() params . append ( param_group ) continue # for norm layers, overwrite the weight decay of weight and bias # TODO: obtain the norm layer prefixes dynamically if re . search ( r'(bn|gn)(\d+)?.(weight|bias)' , name ) : if base_wd is not None : param_group [ 'weight_decay' ] = base_wd * norm_decay_mult # for other layers, overwrite both lr and weight decay of bias elif name . endswith ( '.bias' ) : param_group [ 'lr' ] = base_lr * bias_lr_mult if base_wd is not None : param_group [ 'weight_decay' ] = base_wd * bias_decay_mult # otherwise use the global settings params . append ( param_group ) optimizer_cls = getattr ( torch . optim , optimizer_cfg . pop ( 'type' ) ) return optimizer_cls ( params , * * optimizer_cfg )