(self)
Properly initialize the weights of SRU, following the same
        recipe as:
        Xavier init:
            http://proceedings.mlr.press/v9/glorot10a/glorot10a.pdf
        Kaiming init:
            https://arxiv.org/abs/1502.01852
Properly initialize the weights of SRU, following the same
        recipe as:
        Xavier init:
            http://proceedings.mlr.press/v9/glorot10a/glorot10a.pdf
        Kaiming init:
            https://arxiv.org/abs/1502.01852
Properly initialize the weights of SRU following the same recipe as : Xavier init : http : // proceedings . mlr . press / v9 / glorot10a / glorot10a . pdf Kaiming init : https : // arxiv . org / abs / 1502 . 01852
def reset_parameters(self):
        """Properly initialize the weights of SRU, following the same
        recipe as:
        Xavier init:
            http://proceedings.mlr.press/v9/glorot10a/glorot10a.pdf
        Kaiming init:
            https://arxiv.org/abs/1502.01852

        """
        # initialize bias and scaling constant
        self.bias.data.zero_()
        bias_val, output_size = self.highway_bias, self.output_size
        self.bias.data[output_size:].zero_().add_(bias_val)
        self.scale_x.data[0] = 1
        if self.rescale and self.has_skip_term:
            # scalar used to properly scale the highway output
            scale_val = (1 + math.exp(bias_val) * 2)**0.5
            self.scale_x.data[0] = scale_val

        if self.custom_m is None:
            # initialize weights such that E[w_ij]=0 and Var[w_ij]=1/d
            d = self.weight.size(0)
            val_range = (3.0 / d)**0.5
            self.weight.data.uniform_(-val_range, val_range)
            if self.projection_size > 0:
                val_range = (3.0 / self.weight_proj.size(0))**0.5
                self.weight_proj.data.uniform_(-val_range, val_range)

            # projection matrix as a tensor of size:
            #    (input_size, bidirection, hidden_size, num_matrices)
            w = self.weight.data.view(d, -1, self.hidden_size, self.num_matrices)

            # re-scale weights for dropout and normalized input for better gradient flow
            if self.dropout > 0:
                w[:, :, :, 0].mul_((1 - self.dropout)**0.5)
            if self.rnn_dropout > 0:
                w.mul_((1 - self.rnn_dropout)**0.5)

            # making weights smaller when layer norm is used. need more tests
            if self.layer_norm:
                w.mul_(0.1)
                # self.weight_c.data.mul_(0.25)

            # properly scale the highway output
            if self.rescale and self.has_skip_term and self.num_matrices == 4:
                scale_val = (1 + math.exp(bias_val) * 2)**0.5
                w[:, :, :, 3].mul_(scale_val)
        else:
            if hasattr(self.custom_m, 'reset_parameters'):
                self.custom_m.reset_parameters()
            else:
                warnings.warn("Unable to reset parameters for custom module. "
                              "reset_parameters() method not found for custom module. "
                              + self.custom_m.__class__.__name__)

        if not self.v1:
            # intialize weight_c such that E[w]=0 and Var[w]=1
            if self.weight_c_init is None:
                self.weight_c.data.uniform_(-3.0**0.5, 3.0**0.5)
                self.weight_c.data.mul_(0.5**0.5)
            else:
                self.weight_c.data.uniform_(-self.weight_c_init, self.weight_c_init)

            # rescale weight_c and the weight of sigmoid gates with a factor of sqrt(0.5)
            if self.custom_m is None:
                w[:, :, :, 1].mul_(0.5**0.5)
                w[:, :, :, 2].mul_(0.5**0.5)
        else:
            self.weight_c.data.zero_()
            self.weight_c.requires_grad = False
def reset_parameters ( self ) : # initialize bias and scaling constant self . bias . data . zero_ ( ) bias_val , output_size = self . highway_bias , self . output_size self . bias . data [ output_size : ] . zero_ ( ) . add_ ( bias_val ) self . scale_x . data [ 0 ] = 1 if self . rescale and self . has_skip_term : # scalar used to properly scale the highway output scale_val = ( 1 + math . exp ( bias_val ) * 2 ) ** 0.5 self . scale_x . data [ 0 ] = scale_val if self . custom_m is None : # initialize weights such that E[w_ij]=0 and Var[w_ij]=1/d d = self . weight . size ( 0 ) val_range = ( 3.0 / d ) ** 0.5 self . weight . data . uniform_ ( - val_range , val_range ) if self . projection_size > 0 : val_range = ( 3.0 / self . weight_proj . size ( 0 ) ) ** 0.5 self . weight_proj . data . uniform_ ( - val_range , val_range ) # projection matrix as a tensor of size: #    (input_size, bidirection, hidden_size, num_matrices) w = self . weight . data . view ( d , - 1 , self . hidden_size , self . num_matrices ) # re-scale weights for dropout and normalized input for better gradient flow if self . dropout > 0 : w [ : , : , : , 0 ] . mul_ ( ( 1 - self . dropout ) ** 0.5 ) if self . rnn_dropout > 0 : w . mul_ ( ( 1 - self . rnn_dropout ) ** 0.5 ) # making weights smaller when layer norm is used. need more tests if self . layer_norm : w . mul_ ( 0.1 ) # self.weight_c.data.mul_(0.25) # properly scale the highway output if self . rescale and self . has_skip_term and self . num_matrices == 4 : scale_val = ( 1 + math . exp ( bias_val ) * 2 ) ** 0.5 w [ : , : , : , 3 ] . mul_ ( scale_val ) else : if hasattr ( self . custom_m , 'reset_parameters' ) : self . custom_m . reset_parameters ( ) else : warnings . warn ( "Unable to reset parameters for custom module. " "reset_parameters() method not found for custom module. " + self . custom_m . __class__ . __name__ ) if not self . v1 : # intialize weight_c such that E[w]=0 and Var[w]=1 if self . weight_c_init is None : self . weight_c . data . uniform_ ( - 3.0 ** 0.5 , 3.0 ** 0.5 ) self . weight_c . data . mul_ ( 0.5 ** 0.5 ) else : self . weight_c . data . uniform_ ( - self . weight_c_init , self . weight_c_init ) # rescale weight_c and the weight of sigmoid gates with a factor of sqrt(0.5) if self . custom_m is None : w [ : , : , : , 1 ] . mul_ ( 0.5 ** 0.5 ) w [ : , : , : , 2 ] . mul_ ( 0.5 ** 0.5 ) else : self . weight_c . data . zero_ ( ) self . weight_c . requires_grad = False