(text_filepattern, max_lines)
return ret
Read a vocab file and return a dictionary of token counts.

  Reads a two-column CSV file of tokens and their frequency in a dataset. The
  tokens are presumed to be generated by encode() or the equivalent.

  Args:
    text_filepattern: A pattern matching one or more files.
    max_lines: An integer; maximum total lines to read.

  Returns:
    a dictionary mapping token to count.
Read a vocab file and return a dictionary of token counts.
Read a vocab file and return a dictionary of token counts .
def vocab_token_counts(text_filepattern, max_lines):
  """Read a vocab file and return a dictionary of token counts.

  Reads a two-column CSV file of tokens and their frequency in a dataset. The
  tokens are presumed to be generated by encode() or the equivalent.

  Args:
    text_filepattern: A pattern matching one or more files.
    max_lines: An integer; maximum total lines to read.

  Returns:
    a dictionary mapping token to count.
  """
  ret = {}
  for i, line in enumerate(
      _read_filepattern(text_filepattern, max_lines=max_lines)):
    if "," not in line:
      logging.warning("Malformed vocab line #%d '%s'", i, line)
      continue

    token, count = line.rsplit(",", 1)
    ret[token] = int(count)

  return ret
def vocab_token_counts ( text_filepattern , max_lines ) : ret = { } for i , line in enumerate ( _read_filepattern ( text_filepattern , max_lines = max_lines ) ) : if "," not in line : logging . warning ( "Malformed vocab line #%d '%s'" , i , line ) continue token , count = line . rsplit ( "," , 1 ) ret [ token ] = int ( count ) return ret