(self,
                              token_counts,
                              min_count,
                              num_iterations=4,
                              reserved_tokens=None,
                              max_subtoken_length=None)
Train a SubwordTextEncoder based on a dictionary of word counts.

    Args:
      token_counts: a dictionary of Unicode strings to int.
      min_count: an integer - discard subtokens with lower counts.
      num_iterations: an integer.  how many iterations of refinement.
      reserved_tokens: List of reserved tokens. The global variable
        `RESERVED_TOKENS` must be a prefix of `reserved_tokens`. If this
        argument is `None`, it will use `RESERVED_TOKENS`.
      max_subtoken_length: Maximum length of a subtoken. If this is not set,
        then the runtime and memory use of creating the vocab is quadratic in
        the length of the longest token. If this is set, then it is instead
        O(max_subtoken_length * length of longest token).

    Raises:
      ValueError: if reserved is not 0 or len(RESERVED_TOKENS). In this case, it
        is not clear what the space is being reserved for, or when it will be
        filled in.
Train a SubwordTextEncoder based on a dictionary of word counts.
Train a SubwordTextEncoder based on a dictionary of word counts .
def build_from_token_counts(self,
                              token_counts,
                              min_count,
                              num_iterations=4,
                              reserved_tokens=None,
                              max_subtoken_length=None):
    """Train a SubwordTextEncoder based on a dictionary of word counts.

    Args:
      token_counts: a dictionary of Unicode strings to int.
      min_count: an integer - discard subtokens with lower counts.
      num_iterations: an integer.  how many iterations of refinement.
      reserved_tokens: List of reserved tokens. The global variable
        `RESERVED_TOKENS` must be a prefix of `reserved_tokens`. If this
        argument is `None`, it will use `RESERVED_TOKENS`.
      max_subtoken_length: Maximum length of a subtoken. If this is not set,
        then the runtime and memory use of creating the vocab is quadratic in
        the length of the longest token. If this is set, then it is instead
        O(max_subtoken_length * length of longest token).

    Raises:
      ValueError: if reserved is not 0 or len(RESERVED_TOKENS). In this case, it
        is not clear what the space is being reserved for, or when it will be
        filled in.
    """
    if reserved_tokens is None:
      reserved_tokens = RESERVED_TOKENS
    else:
      # There is not complete freedom in replacing RESERVED_TOKENS.
      for default, proposed in zip(RESERVED_TOKENS, reserved_tokens):
        if default != proposed:
          raise ValueError("RESERVED_TOKENS must be a prefix of "
                           "reserved_tokens.")

    # Initialize the alphabet. Note, this must include reserved tokens or it can
    # result in encoding failures.
    alphabet_tokens = itertools.chain(
        six.iterkeys(token_counts),
        [native_to_unicode(t) for t in reserved_tokens])

    self._init_alphabet_from_tokens(alphabet_tokens)

    # Bootstrap the initial list of subtokens with the characters from the
    # alphabet plus the escaping characters.
    self._init_subtokens_from_list(
        list(self._alphabet), reserved_tokens=reserved_tokens)

    # We build iteratively.  On each iteration, we segment all the words,
    # then count the resulting potential subtokens, keeping the ones
    # with high enough counts for our new vocabulary.
    if min_count < 1:
      min_count = 1
    for i in range(num_iterations):
      logging.info("Iteration %d", i)

      # Collect all substrings of the encoded token that break along current
      # subtoken boundaries.
      subtoken_counts = collections.defaultdict(int)
      for token, count in six.iteritems(token_counts):
        iter_start_time = time.time()
        escaped_token = _escape_token(token, self._alphabet)
        subtokens = self._escaped_token_to_subtoken_strings(escaped_token)
        start = 0
        for subtoken in subtokens:
          last_position = len(escaped_token) + 1
          if max_subtoken_length is not None:
            last_position = min(last_position, start + max_subtoken_length)

          for end in range(start + 1, last_position):
            new_subtoken = escaped_token[start:end]
            subtoken_counts[new_subtoken] += count
          start += len(subtoken)
        iter_time_secs = time.time() - iter_start_time
        if iter_time_secs > 0.1:
          logging.info(
              "Processing token [%s] took {%d} seconds, consider "
              "setting Text2TextProblem.max_subtoken_length to a "
              "smaller value.", token, iter_time_secs)

      # Array of sets of candidate subtoken strings, by length.
      len_to_subtoken_strings = []
      for subtoken_string, count in six.iteritems(subtoken_counts):
        lsub = len(subtoken_string)
        if count >= min_count:
          while len(len_to_subtoken_strings) <= lsub:
            len_to_subtoken_strings.append(set())
          len_to_subtoken_strings[lsub].add(subtoken_string)

      # Consider the candidates longest to shortest, so that if we accept
      # a longer subtoken string, we can decrement the counts of its prefixes.
      new_subtoken_strings = []
      for lsub in range(len(len_to_subtoken_strings) - 1, 0, -1):
        subtoken_strings = len_to_subtoken_strings[lsub]
        for subtoken_string in subtoken_strings:
          count = subtoken_counts[subtoken_string]
          if count >= min_count:
            # Exclude alphabet tokens here, as they must be included later,
            # explicitly, regardless of count.
            if subtoken_string not in self._alphabet:
              new_subtoken_strings.append((count, subtoken_string))
            for l in range(1, lsub):
              subtoken_counts[subtoken_string[:l]] -= count

      # Include the alphabet explicitly to guarantee all strings are encodable.
      new_subtoken_strings.extend(
          (subtoken_counts.get(a, 0), a) for a in self._alphabet)
      new_subtoken_strings.sort(reverse=True)

      # Reinitialize to the candidate vocabulary.
      new_subtoken_strings = [subtoken for _, subtoken in new_subtoken_strings]
      if reserved_tokens:
        escaped_reserved_tokens = [
            _escape_token(native_to_unicode(t), self._alphabet)
            for t in reserved_tokens
        ]
        new_subtoken_strings = escaped_reserved_tokens + new_subtoken_strings

      self._init_subtokens_from_list(new_subtoken_strings)
      logging.info("vocab_size = %d", self.vocab_size)
def build_from_token_counts ( self , token_counts , min_count , num_iterations = 4 , reserved_tokens = None , max_subtoken_length = None ) : if reserved_tokens is None : reserved_tokens = RESERVED_TOKENS else : # There is not complete freedom in replacing RESERVED_TOKENS. for default , proposed in zip ( RESERVED_TOKENS , reserved_tokens ) : if default != proposed : raise ValueError ( "RESERVED_TOKENS must be a prefix of " "reserved_tokens." ) # Initialize the alphabet. Note, this must include reserved tokens or it can # result in encoding failures. alphabet_tokens = itertools . chain ( six . iterkeys ( token_counts ) , [ native_to_unicode ( t ) for t in reserved_tokens ] ) self . _init_alphabet_from_tokens ( alphabet_tokens ) # Bootstrap the initial list of subtokens with the characters from the # alphabet plus the escaping characters. self . _init_subtokens_from_list ( list ( self . _alphabet ) , reserved_tokens = reserved_tokens ) # We build iteratively.  On each iteration, we segment all the words, # then count the resulting potential subtokens, keeping the ones # with high enough counts for our new vocabulary. if min_count < 1 : min_count = 1 for i in range ( num_iterations ) : logging . info ( "Iteration %d" , i ) # Collect all substrings of the encoded token that break along current # subtoken boundaries. subtoken_counts = collections . defaultdict ( int ) for token , count in six . iteritems ( token_counts ) : iter_start_time = time . time ( ) escaped_token = _escape_token ( token , self . _alphabet ) subtokens = self . _escaped_token_to_subtoken_strings ( escaped_token ) start = 0 for subtoken in subtokens : last_position = len ( escaped_token ) + 1 if max_subtoken_length is not None : last_position = min ( last_position , start + max_subtoken_length ) for end in range ( start + 1 , last_position ) : new_subtoken = escaped_token [ start : end ] subtoken_counts [ new_subtoken ] += count start += len ( subtoken ) iter_time_secs = time . time ( ) - iter_start_time if iter_time_secs > 0.1 : logging . info ( "Processing token [%s] took {%d} seconds, consider " "setting Text2TextProblem.max_subtoken_length to a " "smaller value." , token , iter_time_secs ) # Array of sets of candidate subtoken strings, by length. len_to_subtoken_strings = [ ] for subtoken_string , count in six . iteritems ( subtoken_counts ) : lsub = len ( subtoken_string ) if count >= min_count : while len ( len_to_subtoken_strings ) <= lsub : len_to_subtoken_strings . append ( set ( ) ) len_to_subtoken_strings [ lsub ] . add ( subtoken_string ) # Consider the candidates longest to shortest, so that if we accept # a longer subtoken string, we can decrement the counts of its prefixes. new_subtoken_strings = [ ] for lsub in range ( len ( len_to_subtoken_strings ) - 1 , 0 , - 1 ) : subtoken_strings = len_to_subtoken_strings [ lsub ] for subtoken_string in subtoken_strings : count = subtoken_counts [ subtoken_string ] if count >= min_count : # Exclude alphabet tokens here, as they must be included later, # explicitly, regardless of count. if subtoken_string not in self . _alphabet : new_subtoken_strings . append ( ( count , subtoken_string ) ) for l in range ( 1 , lsub ) : subtoken_counts [ subtoken_string [ : l ] ] -= count # Include the alphabet explicitly to guarantee all strings are encodable. new_subtoken_strings . extend ( ( subtoken_counts . get ( a , 0 ) , a ) for a in self . _alphabet ) new_subtoken_strings . sort ( reverse = True ) # Reinitialize to the candidate vocabulary. new_subtoken_strings = [ subtoken for _ , subtoken in new_subtoken_strings ] if reserved_tokens : escaped_reserved_tokens = [ _escape_token ( native_to_unicode ( t ) , self . _alphabet ) for t in reserved_tokens ] new_subtoken_strings = escaped_reserved_tokens + new_subtoken_strings self . _init_subtokens_from_list ( new_subtoken_strings ) logging . info ( "vocab_size = %d" , self . vocab_size )