(self, x)
return h
Input has shape [seq length, batch]
Input has shape [seq length, batch]
Input has shape [ seq length batch ]
def forward(self, x):
        """ Input has shape [seq length, batch] """
        positions = torch.arange(len(x), device=x.device).unsqueeze(-1)
        h = self.tokens_embeddings(x)
        h = h + self.position_embeddings(positions).expand_as(h)
        h = self.dropout(h)

        attn_mask = torch.full((len(x), len(x)), -float('Inf'), device=h.device, dtype=h.dtype)
        attn_mask = torch.triu(attn_mask, diagonal=1)
        for l in range(len(self.layer_norm_1)):
            h = self.layer_norms_1[l](h)
            x, _ = self.attentions[l](h, h, h, attn_mask=attn_mask, need_weights=False)
            x = self.dropout(x)
            x = self.adapters_1[l](x) + x  # Add an adapter after attention
            h = x + h

            h = self.layer_norms_2[l](h)
            x = self.feed_forwards[l](h)
            x = self.dropout(x)
            x = self.adapters_2[l](x) + x  # Add an adapter after feed-forward
            h = x + h
        return h
def forward ( self , x ) : positions = torch . arange ( len ( x ) , device = x . device ) . unsqueeze ( - 1 ) h = self . tokens_embeddings ( x ) h = h + self . position_embeddings ( positions ) . expand_as ( h ) h = self . dropout ( h ) attn_mask = torch . full ( ( len ( x ) , len ( x ) ) , - float ( 'Inf' ) , device = h . device , dtype = h . dtype ) attn_mask = torch . triu ( attn_mask , diagonal = 1 ) for l in range ( len ( self . layer_norm_1 ) ) : h = self . layer_norms_1 [ l ] ( h ) x , _ = self . attentions [ l ] ( h , h , h , attn_mask = attn_mask , need_weights = False ) x = self . dropout ( x ) x = self . adapters_1 [ l ] ( x ) + x # Add an adapter after attention h = x + h h = self . layer_norms_2 [ l ] ( h ) x = self . feed_forwards [ l ] ( h ) x = self . dropout ( x ) x = self . adapters_2 [ l ] ( x ) + x # Add an adapter after feed-forward h = x + h return h