(self, x, attn_mask=None, head_mask=None)
return output
Parameters
        ----------
        x: torch.tensor(bs, seq_length, dim)
        attn_mask: torch.tensor(bs, seq_length)

        Outputs
        -------
        sa_weights: torch.tensor(bs, n_heads, seq_length, seq_length)
            The attention weights
        ffn_output: torch.tensor(bs, seq_length, dim)
            The output of the transformer block contextualization.
Parameters
        ----------
        x: torch.tensor(bs, seq_length, dim)
        attn_mask: torch.tensor(bs, seq_length)
Parameters ---------- x : torch . tensor ( bs seq_length dim ) attn_mask : torch . tensor ( bs seq_length )
def forward(self, x, attn_mask=None, head_mask=None):
        """
        Parameters
        ----------
        x: torch.tensor(bs, seq_length, dim)
        attn_mask: torch.tensor(bs, seq_length)

        Outputs
        -------
        sa_weights: torch.tensor(bs, n_heads, seq_length, seq_length)
            The attention weights
        ffn_output: torch.tensor(bs, seq_length, dim)
            The output of the transformer block contextualization.
        """
        # Self-Attention
        sa_output = self.attention(query=x, key=x, value=x, mask=attn_mask, head_mask=head_mask)
        if self.output_attentions:
            sa_output, sa_weights = sa_output                  # (bs, seq_length, dim), (bs, n_heads, seq_length, seq_length)
        else: # To handle these `output_attention` or `output_hidden_states` cases returning tuples
            assert type(sa_output) == tuple
            sa_output = sa_output[0]
        sa_output = self.sa_layer_norm(sa_output + x)          # (bs, seq_length, dim)

        # Feed Forward Network
        ffn_output = self.ffn(sa_output)                             # (bs, seq_length, dim)
        ffn_output = self.output_layer_norm(ffn_output + sa_output)  # (bs, seq_length, dim)

        output = (ffn_output,)
        if self.output_attentions:
            output = (sa_weights,) + output
        return output
def forward ( self , x , attn_mask = None , head_mask = None ) : # Self-Attention sa_output = self . attention ( query = x , key = x , value = x , mask = attn_mask , head_mask = head_mask ) if self . output_attentions : sa_output , sa_weights = sa_output # (bs, seq_length, dim), (bs, n_heads, seq_length, seq_length) else : # To handle these `output_attention` or `output_hidden_states` cases returning tuples assert type ( sa_output ) == tuple sa_output = sa_output [ 0 ] sa_output = self . sa_layer_norm ( sa_output + x ) # (bs, seq_length, dim) # Feed Forward Network ffn_output = self . ffn ( sa_output ) # (bs, seq_length, dim) ffn_output = self . output_layer_norm ( ffn_output + sa_output ) # (bs, seq_length, dim) output = ( ffn_output , ) if self . output_attentions : output = ( sa_weights , ) + output return output