(self, hidden_size, total_key_depth, total_value_depth, filter_size, num_heads,
                 bias_mask, layer_dropout=0.0, attention_dropout=0.0, relu_dropout=0.0)
Parameters:
            hidden_size: Hidden size
            total_key_depth: Size of last dimension of keys. Must be divisible by num_head
            total_value_depth: Size of last dimension of values. Must be divisible by num_head
            output_depth: Size last dimension of the final output
            filter_size: Hidden size of the middle layer in FFN
            num_heads: Number of attention heads
            bias_mask: Masking tensor to prevent connections to future elements
            layer_dropout: Dropout for this layer
            attention_dropout: Dropout probability after attention (Should be non-zero only during training)
            relu_dropout: Dropout probability after relu in FFN (Should be non-zero only during training)
Parameters:
            hidden_size: Hidden size
            total_key_depth: Size of last dimension of keys. Must be divisible by num_head
            total_value_depth: Size of last dimension of values. Must be divisible by num_head
            output_depth: Size last dimension of the final output
            filter_size: Hidden size of the middle layer in FFN
            num_heads: Number of attention heads
            bias_mask: Masking tensor to prevent connections to future elements
            layer_dropout: Dropout for this layer
            attention_dropout: Dropout probability after attention (Should be non-zero only during training)
            relu_dropout: Dropout probability after relu in FFN (Should be non-zero only during training)
Parameters : hidden_size : Hidden size total_key_depth : Size of last dimension of keys . Must be divisible by num_head total_value_depth : Size of last dimension of values . Must be divisible by num_head output_depth : Size last dimension of the final output filter_size : Hidden size of the middle layer in FFN num_heads : Number of attention heads bias_mask : Masking tensor to prevent connections to future elements layer_dropout : Dropout for this layer attention_dropout : Dropout probability after attention ( Should be non - zero only during training ) relu_dropout : Dropout probability after relu in FFN ( Should be non - zero only during training )
def __init__(self, hidden_size, total_key_depth, total_value_depth, filter_size, num_heads,
                 bias_mask, layer_dropout=0.0, attention_dropout=0.0, relu_dropout=0.0):
        """
        Parameters:
            hidden_size: Hidden size
            total_key_depth: Size of last dimension of keys. Must be divisible by num_head
            total_value_depth: Size of last dimension of values. Must be divisible by num_head
            output_depth: Size last dimension of the final output
            filter_size: Hidden size of the middle layer in FFN
            num_heads: Number of attention heads
            bias_mask: Masking tensor to prevent connections to future elements
            layer_dropout: Dropout for this layer
            attention_dropout: Dropout probability after attention (Should be non-zero only during training)
            relu_dropout: Dropout probability after relu in FFN (Should be non-zero only during training)
        """
        
        super(DecoderLayer, self).__init__()
        
        self.multi_head_attention_dec = MultiHeadAttention(hidden_size, total_key_depth, total_value_depth, 
                                                       hidden_size, num_heads, bias_mask, attention_dropout)

        self.multi_head_attention_enc_dec = MultiHeadAttention(hidden_size, total_key_depth, total_value_depth, 
                                                       hidden_size, num_heads, None, attention_dropout)
        
        self.positionwise_feed_forward = PositionwiseFeedForward(hidden_size, filter_size, hidden_size,
                                                                 layer_config='cc', padding = 'left', 
                                                                 dropout=relu_dropout)
        self.dropout = nn.Dropout(layer_dropout)
        self.layer_norm_mha_dec = LayerNorm(hidden_size)
        self.layer_norm_mha_enc = LayerNorm(hidden_size)
        self.layer_norm_ffn = LayerNorm(hidden_size)
def __init__ ( self , hidden_size , total_key_depth , total_value_depth , filter_size , num_heads , bias_mask , layer_dropout = 0.0 , attention_dropout = 0.0 , relu_dropout = 0.0 ) : super ( DecoderLayer , self ) . __init__ ( ) self . multi_head_attention_dec = MultiHeadAttention ( hidden_size , total_key_depth , total_value_depth , hidden_size , num_heads , bias_mask , attention_dropout ) self . multi_head_attention_enc_dec = MultiHeadAttention ( hidden_size , total_key_depth , total_value_depth , hidden_size , num_heads , None , attention_dropout ) self . positionwise_feed_forward = PositionwiseFeedForward ( hidden_size , filter_size , hidden_size , layer_config = 'cc' , padding = 'left' , dropout = relu_dropout ) self . dropout = nn . Dropout ( layer_dropout ) self . layer_norm_mha_dec = LayerNorm ( hidden_size ) self . layer_norm_mha_enc = LayerNorm ( hidden_size ) self . layer_norm_ffn = LayerNorm ( hidden_size )