(self, tokens)
return ret
Converts a list of tokens to a list of subtoken IDs.

    Args:
      tokens: a list of strings.

    Returns:
      a list of integers in the range [0, vocab_size)
Converts a list of tokens to a list of subtoken IDs.
Converts a list of tokens to a list of subtoken IDs .
def _tokens_to_subtoken_ids(self, tokens):
    """Converts a list of tokens to a list of subtoken IDs.

    Args:
      tokens: a list of strings.

    Returns:
      a list of integers in the range [0, vocab_size)
    """
    ret = []
    for token in tokens:
      ret.extend(self._token_to_subtoken_ids(token))
    return ret
def _tokens_to_subtoken_ids ( self , tokens ) : ret = [ ] for token in tokens : ret . extend ( self . _token_to_subtoken_ids ( token ) ) return ret