(self, subtoken_strings, reserved_tokens=None)
Initialize token information from a list of subtoken strings.

    Args:
      subtoken_strings: a list of subtokens
      reserved_tokens: List of reserved tokens. We must have `reserved_tokens`
        as None or the empty list, or else the global variable `RESERVED_TOKENS`
        must be a prefix of `reserved_tokens`.

    Raises:
      ValueError: if reserved is not 0 or len(RESERVED_TOKENS). In this case, it
        is not clear what the space is being reserved for, or when it will be
        filled in.
Initialize token information from a list of subtoken strings.
Initialize token information from a list of subtoken strings .
def _init_subtokens_from_list(self, subtoken_strings, reserved_tokens=None):
    """Initialize token information from a list of subtoken strings.

    Args:
      subtoken_strings: a list of subtokens
      reserved_tokens: List of reserved tokens. We must have `reserved_tokens`
        as None or the empty list, or else the global variable `RESERVED_TOKENS`
        must be a prefix of `reserved_tokens`.

    Raises:
      ValueError: if reserved is not 0 or len(RESERVED_TOKENS). In this case, it
        is not clear what the space is being reserved for, or when it will be
        filled in.
    """
    if reserved_tokens is None:
      reserved_tokens = []

    if reserved_tokens:
      self._all_subtoken_strings = reserved_tokens + subtoken_strings
    else:
      self._all_subtoken_strings = subtoken_strings

    # we remember the maximum length of any subtoken to avoid having to
    # check arbitrarily long strings.
    self._max_subtoken_len = max([len(s) for s in subtoken_strings])
    self._subtoken_string_to_id = {
        s: i + len(reserved_tokens) for i, s in enumerate(subtoken_strings) if s
    }
    # Initialize the cache to empty.
    self._cache_size = 2**20
    self._cache = [(None, None)] * self._cache_size
def _init_subtokens_from_list ( self , subtoken_strings , reserved_tokens = None ) : if reserved_tokens is None : reserved_tokens = [ ] if reserved_tokens : self . _all_subtoken_strings = reserved_tokens + subtoken_strings else : self . _all_subtoken_strings = subtoken_strings # we remember the maximum length of any subtoken to avoid having to # check arbitrarily long strings. self . _max_subtoken_len = max ( [ len ( s ) for s in subtoken_strings ] ) self . _subtoken_string_to_id = { s : i + len ( reserved_tokens ) for i , s in enumerate ( subtoken_strings ) if s } # Initialize the cache to empty. self . _cache_size = 2 ** 20 self . _cache = [ ( None , None ) ] * self . _cache_size