(self, escaped_token)
return ret
Converts an escaped token string to a list of subtoken strings.

    Args:
      escaped_token: An escaped token as a unicode string.

    Returns:
      A list of subtokens as unicode strings.
Converts an escaped token string to a list of subtoken strings.
Converts an escaped token string to a list of subtoken strings .
def _escaped_token_to_subtoken_strings(self, escaped_token):
    """Converts an escaped token string to a list of subtoken strings.

    Args:
      escaped_token: An escaped token as a unicode string.

    Returns:
      A list of subtokens as unicode strings.
    """
    # NOTE: This algorithm is greedy; it won't necessarily produce the "best"
    # list of subtokens.
    ret = []
    start = 0
    token_len = len(escaped_token)
    while start < token_len:
      for end in range(
          min(token_len, start + self._max_subtoken_len), start, -1):
        subtoken = escaped_token[start:end]
        if subtoken in self._subtoken_string_to_id:
          ret.append(subtoken)
          start = end
          break

      else:  # Did not break
        # If there is no possible encoding of the escaped token then one of the
        # characters in the token is not in the alphabet. This should be
        # impossible and would be indicative of a bug.
        assert False, "Token substring not found in subtoken vocabulary."

    return ret
def _escaped_token_to_subtoken_strings ( self , escaped_token ) : # NOTE: This algorithm is greedy; it won't necessarily produce the "best" # list of subtokens. ret = [ ] start = 0 token_len = len ( escaped_token ) while start < token_len : for end in range ( min ( token_len , start + self . _max_subtoken_len ) , start , - 1 ) : subtoken = escaped_token [ start : end ] if subtoken in self . _subtoken_string_to_id : ret . append ( subtoken ) start = end break else : # Did not break # If there is no possible encoding of the escaped token then one of the # characters in the token is not in the alphabet. This should be # impossible and would be indicative of a bug. assert False , "Token substring not found in subtoken vocabulary." return ret