(data_path, target_path, vocabulary_path,
                      tokenizer=None, normalize_digits=True)
Tokenize data file and turn into token-ids using given vocabulary file.

  This function loads data line-by-line from data_path, calls the above
  sentence_to_token_ids, and saves the result to target_path. See comment
  for sentence_to_token_ids on the details of token-ids format.

  Args:
    data_path: path to the data file in one-sentence-per-line format.
    target_path: path where the file with token-ids will be created.
    vocabulary_path: path to the vocabulary file.
    tokenizer: a function to use to tokenize each sentence;
      if None, basic_tokenizer will be used.
    normalize_digits: Boolean; if true, all digits are replaced by 0s.
Tokenize data file and turn into token-ids using given vocabulary file.
Tokenize data file and turn into token - ids using given vocabulary file .
def data_to_token_ids(data_path, target_path, vocabulary_path,
                      tokenizer=None, normalize_digits=True):
  """Tokenize data file and turn into token-ids using given vocabulary file.

  This function loads data line-by-line from data_path, calls the above
  sentence_to_token_ids, and saves the result to target_path. See comment
  for sentence_to_token_ids on the details of token-ids format.

  Args:
    data_path: path to the data file in one-sentence-per-line format.
    target_path: path where the file with token-ids will be created.
    vocabulary_path: path to the vocabulary file.
    tokenizer: a function to use to tokenize each sentence;
      if None, basic_tokenizer will be used.
    normalize_digits: Boolean; if true, all digits are replaced by 0s.
  """
  if not gfile.Exists(target_path):
    print("Tokenizing data in %s" % data_path)
    vocab, _ = initialize_vocabulary(vocabulary_path)
    with gfile.GFile(data_path, mode="r") as data_file:
      with gfile.GFile(target_path, mode="w") as tokens_file:
        counter = 0
        for line in data_file:
          counter += 1
          if counter % 100000 == 0:
            print("  tokenizing line %d" % counter)
          token_ids = sentence_to_token_ids(line, vocab, tokenizer,
                                            normalize_digits)
          tokens_file.write(" ".join([str(tok) for tok in token_ids]) + "\n")
def data_to_token_ids ( data_path , target_path , vocabulary_path , tokenizer = None , normalize_digits = True ) : if not gfile . Exists ( target_path ) : print ( "Tokenizing data in %s" % data_path ) vocab , _ = initialize_vocabulary ( vocabulary_path ) with gfile . GFile ( data_path , mode = "r" ) as data_file : with gfile . GFile ( target_path , mode = "w" ) as tokens_file : counter = 0 for line in data_file : counter += 1 if counter % 100000 == 0 : print ( "  tokenizing line %d" % counter ) token_ids = sentence_to_token_ids ( line , vocab , tokenizer , normalize_digits ) tokens_file . write ( " " . join ( [ str ( tok ) for tok in token_ids ] ) + "\n" )