(self, x, labels=None, padding_mask=None)
return logits
Input has shape [seq length, batch]
Input has shape [seq length, batch]
Input has shape [ seq length batch ]
def forward(self, x, labels=None, padding_mask=None):
        """ Input has shape [seq length, batch] """
        hidden_states = self.transformer(x, padding_mask)
        logits = self.lm_head(hidden_states)

        if labels is not None:
            shift_logits = logits[:-1] if self.transformer.causal else logits
            shift_labels = labels[1:] if self.transformer.causal else labels
            loss_fct = nn.CrossEntropyLoss(ignore_index=-1)
            loss = loss_fct(shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1))
            return logits, loss

        return logits
def forward ( self , x , labels = None , padding_mask = None ) : hidden_states = self . transformer ( x , padding_mask ) logits = self . lm_head ( hidden_states ) if labels is not None : shift_logits = logits [ : - 1 ] if self . transformer . causal else logits shift_labels = labels [ 1 : ] if self . transformer . causal else labels loss_fct = nn . CrossEntropyLoss ( ignore_index = - 1 ) loss = loss_fct ( shift_logits . view ( - 1 , shift_logits . size ( - 1 ) ) , shift_labels . view ( - 1 ) ) return logits , loss return logits