(self, adapters_dim, embed_dim, hidden_dim, num_embeddings, num_max_positions, num_heads, num_layers, dropout)
Transformer with adapters (small bottleneck layers)
Transformer with adapters (small bottleneck layers)
Transformer with adapters ( small bottleneck layers )
def __init__(self, adapters_dim, embed_dim, hidden_dim, num_embeddings, num_max_positions, num_heads, num_layers, dropout):
        """ Transformer with adapters (small bottleneck layers) """
        super().__init__(embed_dim, hidden_dim, num_embeddings, num_max_positions, num_heads, num_layers, dropout)
        self.adapters_1 = nn.ModuleList()
        self.adapters_2 = nn.ModuleList()
        for _ in range(num_layers):
            self.adapters_1.append(nn.Sequential(nn.Linear(embed_dim, adapters_dim),
                                                 nn.ReLU(),
                                                 nn.Linear(adapters_dim, embed_dim)))
            self.adapters_2.append(nn.Sequential(nn.Linear(embed_dim, adapters_dim),
                                                 nn.ReLU(),
                                                 nn.Linear(adapters_dim, embed_dim)))
def __init__ ( self , adapters_dim , embed_dim , hidden_dim , num_embeddings , num_max_positions , num_heads , num_layers , dropout ) : super ( ) . __init__ ( embed_dim , hidden_dim , num_embeddings , num_max_positions , num_heads , num_layers , dropout ) self . adapters_1 = nn . ModuleList ( ) self . adapters_2 = nn . ModuleList ( ) for _ in range ( num_layers ) : self . adapters_1 . append ( nn . Sequential ( nn . Linear ( embed_dim , adapters_dim ) , nn . ReLU ( ) , nn . Linear ( adapters_dim , embed_dim ) ) ) self . adapters_2 . append ( nn . Sequential ( nn . Linear ( embed_dim , adapters_dim ) , nn . ReLU ( ) , nn . Linear ( adapters_dim , embed_dim ) ) )