code: def _token_counts_from_generator(generator, max_chars, reserved_tokens):
    
    reserved_tokens = list(reserved_tokens) + [_UNDERSCORE_REPLACEMENT]
    tokenizer = text_encoder.Tokenizer(
        alphanum_only=False, reserved_tokens=reserved_tokens)
    num_chars = 0
    token_counts = collections.defaultdict(int)
    for s in generator:
      s = tf.compat.as_text(s)
      if max_chars and (num_chars + len(s)) >= max_chars:
        s = s[:(max_chars - num_chars)]
      tokens = tokenizer.tokenize(s)
      tokens = _prepare_tokens_for_encode(tokens)
      for t in tokens:
        token_counts[t] += 1
      if max_chars:
        num_chars += len(s)
        if num_chars > max_chars:
          break
    return token_counts
docs: Builds token counts from generator.