(cls,
                           target_size,
                           token_counts,
                           min_val,
                           max_val,
                           max_subtoken_length=None,
                           reserved_tokens=None,
                           num_iterations=4)
return bisect(min_val, max_val)
Builds a SubwordTextEncoder that has `vocab_size` near `target_size`.

    Uses simple recursive binary search to find a minimum token count that most
    closely matches the `target_size`.

    Args:
      target_size: Desired vocab_size to approximate.
      token_counts: A dictionary of token counts, mapping string to int.
      min_val: An integer; lower bound for the minimum token count.
      max_val: An integer; upper bound for the minimum token count.
      max_subtoken_length: Maximum length of a subtoken. If this is not set,
        then the runtime and memory use of creating the vocab is quadratic in
        the length of the longest token. If this is set, then it is instead
        O(max_subtoken_length * length of longest token).
      reserved_tokens: List of reserved tokens. The global variable
        `RESERVED_TOKENS` must be a prefix of `reserved_tokens`. If this
        argument is `None`, it will use `RESERVED_TOKENS`.
      num_iterations: An integer; how many iterations of refinement.

    Returns:
      A SubwordTextEncoder instance.

    Raises:
      ValueError: If `min_val` is greater than `max_val`.
Builds a SubwordTextEncoder that has `vocab_size` near `target_size`.
Builds a SubwordTextEncoder that has vocab_size near target_size .
def build_to_target_size(cls,
                           target_size,
                           token_counts,
                           min_val,
                           max_val,
                           max_subtoken_length=None,
                           reserved_tokens=None,
                           num_iterations=4):
    """Builds a SubwordTextEncoder that has `vocab_size` near `target_size`.

    Uses simple recursive binary search to find a minimum token count that most
    closely matches the `target_size`.

    Args:
      target_size: Desired vocab_size to approximate.
      token_counts: A dictionary of token counts, mapping string to int.
      min_val: An integer; lower bound for the minimum token count.
      max_val: An integer; upper bound for the minimum token count.
      max_subtoken_length: Maximum length of a subtoken. If this is not set,
        then the runtime and memory use of creating the vocab is quadratic in
        the length of the longest token. If this is set, then it is instead
        O(max_subtoken_length * length of longest token).
      reserved_tokens: List of reserved tokens. The global variable
        `RESERVED_TOKENS` must be a prefix of `reserved_tokens`. If this
        argument is `None`, it will use `RESERVED_TOKENS`.
      num_iterations: An integer; how many iterations of refinement.

    Returns:
      A SubwordTextEncoder instance.

    Raises:
      ValueError: If `min_val` is greater than `max_val`.
    """
    if min_val > max_val:
      raise ValueError("Lower bound for the minimum token count "
                       "is greater than the upper bound.")
    if target_size < 1:
      raise ValueError("Target size must be positive.")

    if reserved_tokens is None:
      reserved_tokens = RESERVED_TOKENS

    def bisect(min_val, max_val):
      """Bisection to find the right size."""
      present_count = (max_val + min_val) // 2
      logging.info("Trying min_count %d", present_count)
      subtokenizer = cls()
      subtokenizer.build_from_token_counts(
          token_counts,
          present_count,
          num_iterations,
          max_subtoken_length=max_subtoken_length,
          reserved_tokens=reserved_tokens)

      # Being within 1% of the target size is ok.
      is_ok = abs(subtokenizer.vocab_size - target_size) * 100 < target_size
      # If min_val == max_val, we can't do any better than this.
      if is_ok or min_val >= max_val or present_count < 2:
        return subtokenizer

      if subtokenizer.vocab_size > target_size:
        other_subtokenizer = bisect(present_count + 1, max_val)
      else:
        other_subtokenizer = bisect(min_val, present_count - 1)

      if other_subtokenizer is None:
        return subtokenizer

      if (abs(other_subtokenizer.vocab_size - target_size) <
          abs(subtokenizer.vocab_size - target_size)):
        return other_subtokenizer
      return subtokenizer

    return bisect(min_val, max_val)
def build_to_target_size ( cls , target_size , token_counts , min_val , max_val , max_subtoken_length = None , reserved_tokens = None , num_iterations = 4 ) : if min_val > max_val : raise ValueError ( "Lower bound for the minimum token count " "is greater than the upper bound." ) if target_size < 1 : raise ValueError ( "Target size must be positive." ) if reserved_tokens is None : reserved_tokens = RESERVED_TOKENS def bisect ( min_val , max_val ) : """Bisection to find the right size.""" present_count = ( max_val + min_val ) // 2 logging . info ( "Trying min_count %d" , present_count ) subtokenizer = cls ( ) subtokenizer . build_from_token_counts ( token_counts , present_count , num_iterations , max_subtoken_length = max_subtoken_length , reserved_tokens = reserved_tokens ) # Being within 1% of the target size is ok. is_ok = abs ( subtokenizer . vocab_size - target_size ) * 100 < target_size # If min_val == max_val, we can't do any better than this. if is_ok or min_val >= max_val or present_count < 2 : return subtokenizer if subtokenizer . vocab_size > target_size : other_subtokenizer = bisect ( present_count + 1 , max_val ) else : other_subtokenizer = bisect ( min_val , present_count - 1 ) if other_subtokenizer is None : return subtokenizer if ( abs ( other_subtokenizer . vocab_size - target_size ) < abs ( subtokenizer . vocab_size - target_size ) ) : return other_subtokenizer return subtokenizer return bisect ( min_val , max_val )