(self, embedding_size, hidden_size, num_layers, num_heads, total_key_depth, total_value_depth,
                 filter_size, max_length=100, input_dropout=0.0, layer_dropout=0.0, 
                 attention_dropout=0.0, relu_dropout=0.0, use_mask=False)
Parameters:
            embedding_size: Size of embeddings
            hidden_size: Hidden size
            num_layers: Total layers in the Encoder
            num_heads: Number of attention heads
            total_key_depth: Size of last dimension of keys. Must be divisible by num_head
            total_value_depth: Size of last dimension of values. Must be divisible by num_head
            output_depth: Size last dimension of the final output
            filter_size: Hidden size of the middle layer in FFN
            max_length: Max sequence length (required for timing signal)
            input_dropout: Dropout just after embedding
            layer_dropout: Dropout for each layer
            attention_dropout: Dropout probability after attention (Should be non-zero only during training)
            relu_dropout: Dropout probability after relu in FFN (Should be non-zero only during training)
            use_mask: Set to True to turn on future value masking
Parameters:
            embedding_size: Size of embeddings
            hidden_size: Hidden size
            num_layers: Total layers in the Encoder
            num_heads: Number of attention heads
            total_key_depth: Size of last dimension of keys. Must be divisible by num_head
            total_value_depth: Size of last dimension of values. Must be divisible by num_head
            output_depth: Size last dimension of the final output
            filter_size: Hidden size of the middle layer in FFN
            max_length: Max sequence length (required for timing signal)
            input_dropout: Dropout just after embedding
            layer_dropout: Dropout for each layer
            attention_dropout: Dropout probability after attention (Should be non-zero only during training)
            relu_dropout: Dropout probability after relu in FFN (Should be non-zero only during training)
            use_mask: Set to True to turn on future value masking
Parameters : embedding_size : Size of embeddings hidden_size : Hidden size num_layers : Total layers in the Encoder num_heads : Number of attention heads total_key_depth : Size of last dimension of keys . Must be divisible by num_head total_value_depth : Size of last dimension of values . Must be divisible by num_head output_depth : Size last dimension of the final output filter_size : Hidden size of the middle layer in FFN max_length : Max sequence length ( required for timing signal ) input_dropout : Dropout just after embedding layer_dropout : Dropout for each layer attention_dropout : Dropout probability after attention ( Should be non - zero only during training ) relu_dropout : Dropout probability after relu in FFN ( Should be non - zero only during training ) use_mask : Set to True to turn on future value masking
def __init__(self, embedding_size, hidden_size, num_layers, num_heads, total_key_depth, total_value_depth,
                 filter_size, max_length=100, input_dropout=0.0, layer_dropout=0.0, 
                 attention_dropout=0.0, relu_dropout=0.0, use_mask=False):
        """
        Parameters:
            embedding_size: Size of embeddings
            hidden_size: Hidden size
            num_layers: Total layers in the Encoder
            num_heads: Number of attention heads
            total_key_depth: Size of last dimension of keys. Must be divisible by num_head
            total_value_depth: Size of last dimension of values. Must be divisible by num_head
            output_depth: Size last dimension of the final output
            filter_size: Hidden size of the middle layer in FFN
            max_length: Max sequence length (required for timing signal)
            input_dropout: Dropout just after embedding
            layer_dropout: Dropout for each layer
            attention_dropout: Dropout probability after attention (Should be non-zero only during training)
            relu_dropout: Dropout probability after relu in FFN (Should be non-zero only during training)
            use_mask: Set to True to turn on future value masking
        """
        
        super(Encoder, self).__init__()
        
        self.timing_signal = _gen_timing_signal(max_length, hidden_size)
        
        params =(hidden_size, 
                 total_key_depth or hidden_size,
                 total_value_depth or hidden_size,
                 filter_size, 
                 num_heads, 
                 _gen_bias_mask(max_length) if use_mask else None,
                 layer_dropout, 
                 attention_dropout, 
                 relu_dropout)
        
        self.embedding_proj = nn.Linear(embedding_size, hidden_size, bias=False)
        self.enc = nn.Sequential(*[EncoderLayer(*params) for l in range(num_layers)])
        
        self.layer_norm = LayerNorm(hidden_size)
        self.input_dropout = nn.Dropout(input_dropout)
def __init__ ( self , embedding_size , hidden_size , num_layers , num_heads , total_key_depth , total_value_depth , filter_size , max_length = 100 , input_dropout = 0.0 , layer_dropout = 0.0 , attention_dropout = 0.0 , relu_dropout = 0.0 , use_mask = False ) : super ( Encoder , self ) . __init__ ( ) self . timing_signal = _gen_timing_signal ( max_length , hidden_size ) params = ( hidden_size , total_key_depth or hidden_size , total_value_depth or hidden_size , filter_size , num_heads , _gen_bias_mask ( max_length ) if use_mask else None , layer_dropout , attention_dropout , relu_dropout ) self . embedding_proj = nn . Linear ( embedding_size , hidden_size , bias = False ) self . enc = nn . Sequential ( * [ EncoderLayer ( * params ) for l in range ( num_layers ) ] ) self . layer_norm = LayerNorm ( hidden_size ) self . input_dropout = nn . Dropout ( input_dropout )