(self, tokens)
return ids
Converts a sequence of tokens into ids using the vocab.
Converts a sequence of tokens into ids using the vocab.
Converts a sequence of tokens into ids using the vocab .
def convert_tokens_to_ids(self, tokens):
        """Converts a sequence of tokens into ids using the vocab."""
        ids = []
        for token in tokens:
            ids.append(self.vocab[token])
        return ids
def convert_tokens_to_ids ( self , tokens ) : ids = [ ] for token in tokens : ids . append ( self . vocab [ token ] ) return ids