(source_path, target_path, config, max_size=None)
return data_set
Read data from source and target files and put into buckets.

  Args:
    source_path: path to the files with token-ids for the source language.
    target_path: path to the file with token-ids for the target language;
      it must be aligned with the source file: n-th line contains the desired
      output for n-th line from the source_path.
    max_size: maximum number of lines to read, all other will be ignored;
      if 0 or None, data files will be read completely (no limit).

  Returns:
    data_set: a list of length len(config.buckets); data_set[n] contains a list of
      (source, target) pairs read from the provided data files that fit
      into the n-th bucket, i.e., such that len(source) < config.buckets[n][0] and
      len(target) < config.buckets[n][1]; source and target are lists of token-ids.
Read data from source and target files and put into buckets.
Read data from source and target files and put into buckets .
def read_data(source_path, target_path, config, max_size=None):
  """Read data from source and target files and put into buckets.

  Args:
    source_path: path to the files with token-ids for the source language.
    target_path: path to the file with token-ids for the target language;
      it must be aligned with the source file: n-th line contains the desired
      output for n-th line from the source_path.
    max_size: maximum number of lines to read, all other will be ignored;
      if 0 or None, data files will be read completely (no limit).

  Returns:
    data_set: a list of length len(config.buckets); data_set[n] contains a list of
      (source, target) pairs read from the provided data files that fit
      into the n-th bucket, i.e., such that len(source) < config.buckets[n][0] and
      len(target) < config.buckets[n][1]; source and target are lists of token-ids.
  """
  data_set = [[] for _ in config.buckets]
  with tf.gfile.GFile(source_path, mode="r") as source_file:
    with tf.gfile.GFile(target_path, mode="r") as target_file:
      source, target = source_file.readline(), target_file.readline()
      counter = 0
      while source and target and (not max_size or counter < max_size):
        counter += 1
        if counter % 100000 == 0:
          print("  reading data line %d" % counter)
          sys.stdout.flush()
        source_ids = [int(x) for x in source.split()]
        target_ids = [int(x) for x in target.split()]
        target_ids.append(data_utils.EOS_ID)
        for bucket_id, (source_size, target_size) in enumerate(config.buckets):
          if len(source_ids) < source_size and len(target_ids) < target_size:
            data_set[bucket_id].append([source_ids, target_ids])
            break
        source, target = source_file.readline(), target_file.readline()
  return data_set
def read_data ( source_path , target_path , config , max_size = None ) : data_set = [ [ ] for _ in config . buckets ] with tf . gfile . GFile ( source_path , mode = "r" ) as source_file : with tf . gfile . GFile ( target_path , mode = "r" ) as target_file : source , target = source_file . readline ( ) , target_file . readline ( ) counter = 0 while source and target and ( not max_size or counter < max_size ) : counter += 1 if counter % 100000 == 0 : print ( "  reading data line %d" % counter ) sys . stdout . flush ( ) source_ids = [ int ( x ) for x in source . split ( ) ] target_ids = [ int ( x ) for x in target . split ( ) ] target_ids . append ( data_utils . EOS_ID ) for bucket_id , ( source_size , target_size ) in enumerate ( config . buckets ) : if len ( source_ids ) < source_size and len ( target_ids ) < target_size : data_set [ bucket_id ] . append ( [ source_ids , target_ids ] ) break source , target = source_file . readline ( ) , target_file . readline ( ) return data_set