(self, config)
Transformer with a language modeling head on top (tied weights)
Transformer with a language modeling head on top (tied weights)
Transformer with a language modeling head on top ( tied weights )
def __init__(self, config):
        """ Transformer with a language modeling head on top (tied weights) """
        super().__init__()
        self.config = config
        self.transformer = Transformer(config.embed_dim, config.hidden_dim, config.num_embeddings,
                                       config.num_max_positions, config.num_heads, config.num_layers,
                                       config.dropout, config.sinusoidal_embeddings, causal=not config.mlm)
        self.lm_head = nn.Linear(config.embed_dim, config.num_embeddings, bias=False)
        self.apply(self.init_weights)
        self.tie_weights()
def __init__ ( self , config ) : super ( ) . __init__ ( ) self . config = config self . transformer = Transformer ( config . embed_dim , config . hidden_dim , config . num_embeddings , config . num_max_positions , config . num_heads , config . num_layers , config . dropout , config . sinusoidal_embeddings , causal = not config . mlm ) self . lm_head = nn . Linear ( config . embed_dim , config . num_embeddings , bias = False ) self . apply ( self . init_weights ) self . tie_weights ( )