(self, x, clf_tokens_mask, lm_labels=None, clf_labels=None, padding_mask=None)
return lm_logits, clf_logits
Input has shape [seq length, batch]
Input has shape [seq length, batch]
Input has shape [ seq length batch ]
def forward(self, x, clf_tokens_mask, lm_labels=None, clf_labels=None, padding_mask=None):
        """ Input has shape [seq length, batch] """
        hidden_states = self.transformer(x, padding_mask)
        lm_logits = self.lm_head(hidden_states)
        clf_tokens_states = (hidden_states * clf_tokens_mask.unsqueeze(-1).float()).sum(dim=0)
        clf_logits = self.classification_head(clf_tokens_states)

        loss = []
        if clf_labels is not None:
            loss_fct = nn.CrossEntropyLoss(ignore_index=-1)
            loss.append(loss_fct(clf_logits.view(-1, clf_logits.size(-1)), clf_labels.view(-1)))

        if lm_labels is not None:
            shift_logits = lm_logits[:-1] if self.transformer.causal else lm_logits
            shift_labels = lm_labels[1:] if self.transformer.causal else lm_labels
            loss_fct = nn.CrossEntropyLoss(ignore_index=-1)
            loss.append(loss_fct(shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1)))

        if len(loss):
            return (lm_logits, clf_logits), loss

        return lm_logits, clf_logits
def forward ( self , x , clf_tokens_mask , lm_labels = None , clf_labels = None , padding_mask = None ) : hidden_states = self . transformer ( x , padding_mask ) lm_logits = self . lm_head ( hidden_states ) clf_tokens_states = ( hidden_states * clf_tokens_mask . unsqueeze ( - 1 ) . float ( ) ) . sum ( dim = 0 ) clf_logits = self . classification_head ( clf_tokens_states ) loss = [ ] if clf_labels is not None : loss_fct = nn . CrossEntropyLoss ( ignore_index = - 1 ) loss . append ( loss_fct ( clf_logits . view ( - 1 , clf_logits . size ( - 1 ) ) , clf_labels . view ( - 1 ) ) ) if lm_labels is not None : shift_logits = lm_logits [ : - 1 ] if self . transformer . causal else lm_logits shift_labels = lm_labels [ 1 : ] if self . transformer . causal else lm_labels loss_fct = nn . CrossEntropyLoss ( ignore_index = - 1 ) loss . append ( loss_fct ( shift_logits . view ( - 1 , shift_logits . size ( - 1 ) ) , shift_labels . view ( - 1 ) ) ) if len ( loss ) : return ( lm_logits , clf_logits ) , loss return lm_logits , clf_logits