(self, x)
return x.permute(0, 2, 1, 3)
From this,
From this,
From this
def transpose_for_scores(self, x):
        """ From this,

        """
        # Maybe: x = [B, seq_len, all_head_size=hidden_size ]
        # [B, seq_len] + [num_attention_heads, attention_head_size] ???
        new_x_shape = x.size()[:-1] + (self.num_attention_heads, self.attention_head_size) # it is append (for torch.Size tensor, + acts as an concat. operation)

        # [B, seq_len, all_head_size=hidden_size ] -> [B, seq_len, num_attention_heads, attention_head_size]
        x = x.view(*new_x_shape)

        # [B, seq_len, num_attention_heads, attention_head_size] -> [B, num_attention_heads, seq_len, attention_head_size]
        return x.permute(0, 2, 1, 3)
def transpose_for_scores ( self , x ) : # Maybe: x = [B, seq_len, all_head_size=hidden_size ] # [B, seq_len] + [num_attention_heads, attention_head_size] ??? new_x_shape = x . size ( ) [ : - 1 ] + ( self . num_attention_heads , self . attention_head_size ) # it is append (for torch.Size tensor, + acts as an concat. operation) # [B, seq_len, all_head_size=hidden_size ] -> [B, seq_len, num_attention_heads, attention_head_size] x = x . view ( * new_x_shape ) # [B, seq_len, num_attention_heads, attention_head_size] -> [B, num_attention_heads, seq_len, attention_head_size] return x . permute ( 0 , 2 , 1 , 3 )