(self, input_depth, total_key_depth, total_value_depth, output_depth, 
                 num_heads, bias_mask=None, dropout=0.0)
Parameters:
            input_depth: Size of last dimension of input
            total_key_depth: Size of last dimension of keys. Must be divisible by num_head
            total_value_depth: Size of last dimension of values. Must be divisible by num_head
            output_depth: Size last dimension of the final output
            num_heads: Number of attention heads
            bias_mask: Masking tensor to prevent connections to future elements
            dropout: Dropout probability (Should be non-zero only during training)
Parameters:
            input_depth: Size of last dimension of input
            total_key_depth: Size of last dimension of keys. Must be divisible by num_head
            total_value_depth: Size of last dimension of values. Must be divisible by num_head
            output_depth: Size last dimension of the final output
            num_heads: Number of attention heads
            bias_mask: Masking tensor to prevent connections to future elements
            dropout: Dropout probability (Should be non-zero only during training)
Parameters : input_depth : Size of last dimension of input total_key_depth : Size of last dimension of keys . Must be divisible by num_head total_value_depth : Size of last dimension of values . Must be divisible by num_head output_depth : Size last dimension of the final output num_heads : Number of attention heads bias_mask : Masking tensor to prevent connections to future elements dropout : Dropout probability ( Should be non - zero only during training )
def __init__(self, input_depth, total_key_depth, total_value_depth, output_depth, 
                 num_heads, bias_mask=None, dropout=0.0):
        """
        Parameters:
            input_depth: Size of last dimension of input
            total_key_depth: Size of last dimension of keys. Must be divisible by num_head
            total_value_depth: Size of last dimension of values. Must be divisible by num_head
            output_depth: Size last dimension of the final output
            num_heads: Number of attention heads
            bias_mask: Masking tensor to prevent connections to future elements
            dropout: Dropout probability (Should be non-zero only during training)
        """
        super(MultiHeadAttention, self).__init__()
        # Checks borrowed from 
        # https://github.com/tensorflow/tensor2tensor/blob/master/tensor2tensor/layers/common_attention.py
        if total_key_depth % num_heads != 0:
            raise ValueError("Key depth (%d) must be divisible by the number of "
                             "attention heads (%d)." % (total_key_depth, num_heads))
        if total_value_depth % num_heads != 0:
            raise ValueError("Value depth (%d) must be divisible by the number of "
                             "attention heads (%d)." % (total_value_depth, num_heads))
            
        self.num_heads = num_heads
        self.query_scale = (total_key_depth//num_heads)**-0.5
        self.bias_mask = bias_mask
        
        # Key and query depth will be same
        self.query_linear = nn.Linear(input_depth, total_key_depth, bias=False)
        self.key_linear = nn.Linear(input_depth, total_key_depth, bias=False)
        self.value_linear = nn.Linear(input_depth, total_value_depth, bias=False)
        self.output_linear = nn.Linear(total_value_depth, output_depth, bias=False)
        
        self.dropout = nn.Dropout(dropout)
def __init__ ( self , input_depth , total_key_depth , total_value_depth , output_depth , num_heads , bias_mask = None , dropout = 0.0 ) : super ( MultiHeadAttention , self ) . __init__ ( ) # Checks borrowed from  # https://github.com/tensorflow/tensor2tensor/blob/master/tensor2tensor/layers/common_attention.py if total_key_depth % num_heads != 0 : raise ValueError ( "Key depth (%d) must be divisible by the number of " "attention heads (%d)." % ( total_key_depth , num_heads ) ) if total_value_depth % num_heads != 0 : raise ValueError ( "Value depth (%d) must be divisible by the number of " "attention heads (%d)." % ( total_value_depth , num_heads ) ) self . num_heads = num_heads self . query_scale = ( total_key_depth // num_heads ) ** - 0.5 self . bias_mask = bias_mask # Key and query depth will be same self . query_linear = nn . Linear ( input_depth , total_key_depth , bias = False ) self . key_linear = nn . Linear ( input_depth , total_key_depth , bias = False ) self . value_linear = nn . Linear ( input_depth , total_value_depth , bias = False ) self . output_linear = nn . Linear ( total_value_depth , output_depth , bias = False ) self . dropout = nn . Dropout ( dropout )