code: def vocab_token_counts(text_filepattern, max_lines):
    
    ret = {}
    for i, line in enumerate(
        _read_filepattern(text_filepattern, max_lines=max_lines)):
      if "," not in line:
        tf.logging.warning("Malformed vocab line #%d '%s'", i, line)
        continue
  
      token, count = line.rsplit(",", 1)
      ret[_native_to_unicode(token)] = int(count)
  
    return ret
docs: Read a vocab file and return a dictionary of token counts.
  
    Reads a two-column CSV file of tokens and their frequency in a dataset. The
    tokens are presumed to be generated by encode() or the equivalent.
  
    Args:
      text_filepattern: A pattern matching one or more files.
      max_lines: An integer; maximum total lines to read.
  
    Returns:
      a dictionary mapping token to count.