(self, token_text)
return self._tokens_to_subtoken_ids([native_to_unicode(token_text)])
Converts string to list of subtoken IDs without calling tokenizer.

    This treats `token_text` as a single token and directly converts it
    to subtoken IDs. This may be useful when the default tokenizer doesn't
    do what we want (e.g., when encoding text with tokens composed of lots of
    nonalphanumeric characters). It is then up to the caller to make sure that
    raw text is consistently converted into tokens. Only use this if you are
    sure that `encode` doesn't suit your needs.

    Args:
      token_text: A native string representation of a single token.

    Returns:
      A list of subword token IDs; i.e., integers in the range [0, vocab_size).
Converts string to list of subtoken IDs without calling tokenizer.
Converts string to list of subtoken IDs without calling tokenizer .
def encode_without_tokenizing(self, token_text):
    """Converts string to list of subtoken IDs without calling tokenizer.

    This treats `token_text` as a single token and directly converts it
    to subtoken IDs. This may be useful when the default tokenizer doesn't
    do what we want (e.g., when encoding text with tokens composed of lots of
    nonalphanumeric characters). It is then up to the caller to make sure that
    raw text is consistently converted into tokens. Only use this if you are
    sure that `encode` doesn't suit your needs.

    Args:
      token_text: A native string representation of a single token.

    Returns:
      A list of subword token IDs; i.e., integers in the range [0, vocab_size).
    """
    return self._tokens_to_subtoken_ids([native_to_unicode(token_text)])
def encode_without_tokenizing ( self , token_text ) : return self . _tokens_to_subtoken_ids ( [ native_to_unicode ( token_text ) ] )