(self, embed_dim, hidden_dim, num_embeddings, num_max_positions, num_heads, num_layers, dropout,
                 sinusoidal_embeddings, causal=False)
Transformer (GPT-2 architecture)
Transformer (GPT-2 architecture)
Transformer ( GPT - 2 architecture )
def __init__(self, embed_dim, hidden_dim, num_embeddings, num_max_positions, num_heads, num_layers, dropout,
                 sinusoidal_embeddings, causal=False):
        """ Transformer (GPT-2 architecture) """
        super().__init__()
        self.causal = causal
        self.tokens_embeddings = nn.Embedding(num_embeddings, embed_dim)
        self.position_embeddings = nn.Embedding(num_max_positions, embed_dim)
        if sinusoidal_embeddings:
            create_sinusoidal_embeddings(self.position_embeddings)
        self.dropout = nn.Dropout(dropout)

        self.attentions, self.feed_forwards = nn.ModuleList(), nn.ModuleList()
        self.layer_norms_1, self.layer_norms_2 = nn.ModuleList(), nn.ModuleList()
        for _ in range(num_layers):
            self.attentions.append(nn.MultiheadAttention(embed_dim, num_heads, dropout=dropout))
            self.feed_forwards.append(nn.Sequential(nn.Linear(embed_dim, hidden_dim),
                                                    nn.ReLU(),
                                                    nn.Linear(hidden_dim, embed_dim)))
            self.layer_norms_1.append(nn.LayerNorm(embed_dim, eps=1e-12))
            self.layer_norms_2.append(nn.LayerNorm(embed_dim, eps=1e-12))
def __init__ ( self , embed_dim , hidden_dim , num_embeddings , num_max_positions , num_heads , num_layers , dropout , sinusoidal_embeddings , causal = False ) : super ( ) . __init__ ( ) self . causal = causal self . tokens_embeddings = nn . Embedding ( num_embeddings , embed_dim ) self . position_embeddings = nn . Embedding ( num_max_positions , embed_dim ) if sinusoidal_embeddings : create_sinusoidal_embeddings ( self . position_embeddings ) self . dropout = nn . Dropout ( dropout ) self . attentions , self . feed_forwards = nn . ModuleList ( ) , nn . ModuleList ( ) self . layer_norms_1 , self . layer_norms_2 = nn . ModuleList ( ) , nn . ModuleList ( ) for _ in range ( num_layers ) : self . attentions . append ( nn . MultiheadAttention ( embed_dim , num_heads , dropout = dropout ) ) self . feed_forwards . append ( nn . Sequential ( nn . Linear ( embed_dim , hidden_dim ) , nn . ReLU ( ) , nn . Linear ( hidden_dim , embed_dim ) ) ) self . layer_norms_1 . append ( nn . LayerNorm ( embed_dim , eps = 1e-12 ) ) self . layer_norms_2 . append ( nn . LayerNorm ( embed_dim , eps = 1e-12 ) )