(vocab, tokens)
return ids
Converts a sequence of tokens into ids using the vocab.
Converts a sequence of tokens into ids using the vocab.
Converts a sequence of tokens into ids using the vocab .
def convert_tokens_to_ids(vocab, tokens):
    """Converts a sequence of tokens into ids using the vocab."""
    ids = []
    for token in tokens:
        ids.append(vocab[token])
    return ids
def convert_tokens_to_ids ( vocab , tokens ) : ids = [ ] for token in tokens : ids . append ( vocab [ token ] ) return ids