docs: def _count_and_gen_subtokens(
      token_counts, alphabet, subtoken_dict, max_subtoken_length):
    """Count number of times subtokens appear, and generate new subtokens.
  
    Args:
      token_counts: dict mapping tokens to the number of times they appear in the
        original files.
      alphabet: list of allowed characters. Used to escape the tokens, which
        guarantees that all tokens can be split into subtokens.
      subtoken_dict: dict mapping subtokens to ids.
      max_subtoken_length: maximum length of subtoken in subtoken_dict.
  
    Returns:
      A defaultdict mapping subtokens to the number of times they appear in the
      tokens. The dict may contain new subtokens.
    """
    subtoken_counts = collections.defaultdict(int)
    for token, count in six.iteritems(token_counts):
      token = _escape_token(token, alphabet)
      subtokens = _split_token_to_subtokens(
          token, subtoken_dict, max_subtoken_length)
  
      # Generate new subtokens by taking substrings from token.
      start = 0
      for subtoken in subtokens:
        for end in xrange(start + 1, len(token) + 1):
          new_subtoken = token[start:end]
          subtoken_counts[new_subtoken] += count
        start += len(subtoken)
  
    return subtoken_counts