(self, x, attn_mask=None, head_mask=None)
return outputs
Parameters
        ----------
        x: torch.tensor(bs, seq_length, dim)
            Input sequence embedded.
        attn_mask: torch.tensor(bs, seq_length)
            Attention mask on the sequence.

        Outputs
        -------
        hidden_state: torch.tensor(bs, seq_length, dim)
            Sequence of hiddens states in the last (top) layer
        all_hidden_states: Tuple[torch.tensor(bs, seq_length, dim)]
            Tuple of length n_layers with the hidden states from each layer.
            Optional: only if output_hidden_states=True
        all_attentions: Tuple[torch.tensor(bs, n_heads, seq_length, seq_length)]
            Tuple of length n_layers with the attention weights from each layer
            Optional: only if output_attentions=True
Parameters
        ----------
        x: torch.tensor(bs, seq_length, dim)
            Input sequence embedded.
        attn_mask: torch.tensor(bs, seq_length)
            Attention mask on the sequence.
Parameters ---------- x : torch . tensor ( bs seq_length dim ) Input sequence embedded . attn_mask : torch . tensor ( bs seq_length ) Attention mask on the sequence .
def forward(self, x, attn_mask=None, head_mask=None):
        """
        Parameters
        ----------
        x: torch.tensor(bs, seq_length, dim)
            Input sequence embedded.
        attn_mask: torch.tensor(bs, seq_length)
            Attention mask on the sequence.

        Outputs
        -------
        hidden_state: torch.tensor(bs, seq_length, dim)
            Sequence of hiddens states in the last (top) layer
        all_hidden_states: Tuple[torch.tensor(bs, seq_length, dim)]
            Tuple of length n_layers with the hidden states from each layer.
            Optional: only if output_hidden_states=True
        all_attentions: Tuple[torch.tensor(bs, n_heads, seq_length, seq_length)]
            Tuple of length n_layers with the attention weights from each layer
            Optional: only if output_attentions=True
        """
        all_hidden_states = ()
        all_attentions = ()

        hidden_state = x
        for i, layer_module in enumerate(self.layer):
            if self.output_hidden_states:
                all_hidden_states = all_hidden_states + (hidden_state,)

            layer_outputs = layer_module(x=hidden_state,
                                         attn_mask=attn_mask,
                                         head_mask=head_mask[i])
            hidden_state = layer_outputs[-1]

            if self.output_attentions:
                assert len(layer_outputs) == 2
                attentions = layer_outputs[0]
                all_attentions = all_attentions + (attentions,)
            else:
                assert len(layer_outputs) == 1

        # Add last layer
        if self.output_hidden_states:
            all_hidden_states = all_hidden_states + (hidden_state,)

        outputs = (hidden_state,)
        if self.output_hidden_states:
            outputs = outputs + (all_hidden_states,)
        if self.output_attentions:
            outputs = outputs + (all_attentions,)
        return outputs
def forward ( self , x , attn_mask = None , head_mask = None ) : all_hidden_states = ( ) all_attentions = ( ) hidden_state = x for i , layer_module in enumerate ( self . layer ) : if self . output_hidden_states : all_hidden_states = all_hidden_states + ( hidden_state , ) layer_outputs = layer_module ( x = hidden_state , attn_mask = attn_mask , head_mask = head_mask [ i ] ) hidden_state = layer_outputs [ - 1 ] if self . output_attentions : assert len ( layer_outputs ) == 2 attentions = layer_outputs [ 0 ] all_attentions = all_attentions + ( attentions , ) else : assert len ( layer_outputs ) == 1 # Add last layer if self . output_hidden_states : all_hidden_states = all_hidden_states + ( hidden_state , ) outputs = ( hidden_state , ) if self . output_hidden_states : outputs = outputs + ( all_hidden_states , ) if self . output_attentions : outputs = outputs + ( all_attentions , ) return outputs