(self, inputs)
return y, encoder_outputs
NOTE: Inputs is a tuple consisting of decoder inputs and encoder output
NOTE: Inputs is a tuple consisting of decoder inputs and encoder output
NOTE : Inputs is a tuple consisting of decoder inputs and encoder output
def forward(self, inputs):
        """
        NOTE: Inputs is a tuple consisting of decoder inputs and encoder output
        """
        x, encoder_outputs = inputs
        
        # Layer Normalization before decoder self attention
        x_norm = self.layer_norm_mha_dec(x)
        
        # Masked Multi-head attention
        y = self.multi_head_attention_dec(x_norm, x_norm, x_norm)
        
        # Dropout and residual after self-attention
        x = self.dropout(x + y)

        # Layer Normalization before encoder-decoder attention
        x_norm = self.layer_norm_mha_enc(x)

        # Multi-head encoder-decoder attention
        y = self.multi_head_attention_enc_dec(x_norm, encoder_outputs, encoder_outputs)
        
        # Dropout and residual after encoder-decoder attention
        x = self.dropout(x + y)
        
        # Layer Normalization
        x_norm = self.layer_norm_ffn(x)
        
        # Positionwise Feedforward
        y = self.positionwise_feed_forward(x_norm)
        
        # Dropout and residual after positionwise feed forward layer
        y = self.dropout(x + y)
        
        # Return encoder outputs as well to work with nn.Sequential
        return y, encoder_outputs
def forward ( self , inputs ) : x , encoder_outputs = inputs # Layer Normalization before decoder self attention x_norm = self . layer_norm_mha_dec ( x ) # Masked Multi-head attention y = self . multi_head_attention_dec ( x_norm , x_norm , x_norm ) # Dropout and residual after self-attention x = self . dropout ( x + y ) # Layer Normalization before encoder-decoder attention x_norm = self . layer_norm_mha_enc ( x ) # Multi-head encoder-decoder attention y = self . multi_head_attention_enc_dec ( x_norm , encoder_outputs , encoder_outputs ) # Dropout and residual after encoder-decoder attention x = self . dropout ( x + y ) # Layer Normalization x_norm = self . layer_norm_ffn ( x ) # Positionwise Feedforward y = self . positionwise_feed_forward ( x_norm ) # Dropout and residual after positionwise feed forward layer y = self . dropout ( x + y ) # Return encoder outputs as well to work with nn.Sequential return y , encoder_outputs