(thread_index, ranges, name, images, decoder, vocab,
                         num_shards)
Processes and saves a subset of images as TFRecord files in one thread.
    Args:
      thread_index: Integer thread identifier within [0, len(ranges)].
      ranges: A list of pairs of integers specifying the ranges of the dataset to
        process in parallel.
      name: Unique identifier specifying the dataset.
      images: List of ImageMetadata.
      decoder: An ImageDecoder object.
      vocab: A Vocabulary object.
      num_shards: Integer number of shards for the output files.
Processes and saves a subset of images as TFRecord files in one thread.
    Args:
      thread_index: Integer thread identifier within [0, len(ranges)].
      ranges: A list of pairs of integers specifying the ranges of the dataset to
        process in parallel.
      name: Unique identifier specifying the dataset.
      images: List of ImageMetadata.
      decoder: An ImageDecoder object.
      vocab: A Vocabulary object.
      num_shards: Integer number of shards for the output files.
Processes and saves a subset of images as TFRecord files in one thread . Args : thread_index : Integer thread identifier within [ 0 len ( ranges ) ] . ranges : A list of pairs of integers specifying the ranges of the dataset to process in parallel . name : Unique identifier specifying the dataset . images : List of ImageMetadata . decoder : An ImageDecoder object . vocab : A Vocabulary object . num_shards : Integer number of shards for the output files .
def _process_image_files(thread_index, ranges, name, images, decoder, vocab,
                         num_shards):
    """Processes and saves a subset of images as TFRecord files in one thread.
    Args:
      thread_index: Integer thread identifier within [0, len(ranges)].
      ranges: A list of pairs of integers specifying the ranges of the dataset to
        process in parallel.
      name: Unique identifier specifying the dataset.
      images: List of ImageMetadata.
      decoder: An ImageDecoder object.
      vocab: A Vocabulary object.
      num_shards: Integer number of shards for the output files.
    """
    # Each thread produces N shards where N = num_shards / num_threads. For
    # instance, if num_shards = 128, and num_threads = 2, then the first thread
    # would produce shards [0, 64).
    num_threads = len(ranges)
    assert not num_shards % num_threads
    num_shards_per_batch = int(num_shards / num_threads)

    shard_ranges = np.linspace(ranges[thread_index][0], ranges[thread_index][1],
                               num_shards_per_batch + 1).astype(int)
    num_images_in_thread = ranges[thread_index][1] - ranges[thread_index][0]

    counter = 0
    for s in range(num_shards_per_batch):
        # Generate a sharded version of the file name, e.g. 'train-00002-of-00010'
        shard = thread_index * num_shards_per_batch + s
        output_filename = "%s-%.5d-of-%.5d" % (name, shard, num_shards)
        output_file = os.path.join(FLAGS.output_dir, output_filename)
        writer = tf.python_io.TFRecordWriter(output_file)

        shard_counter = 0
        images_in_shard = np.arange(shard_ranges[s], shard_ranges[s + 1], dtype=int)
        for i in images_in_shard:
            image = images[i]

            sequence_example = _to_sequence_example(image, decoder, vocab)
            if sequence_example is not None:
                writer.write(sequence_example.SerializeToString())
                shard_counter += 1
                counter += 1

            if not counter % 1000:
                print("%s [thread %d]: Processed %d of %d items in thread batch." %
                      (datetime.now(), thread_index, counter, num_images_in_thread))
                sys.stdout.flush()

        writer.close()
        print("%s [thread %d]: Wrote %d image-caption pairs to %s" %
              (datetime.now(), thread_index, shard_counter, output_file))
        sys.stdout.flush()
        shard_counter = 0
    print("%s [thread %d]: Wrote %d image-caption pairs to %d shards." %
          (datetime.now(), thread_index, counter, num_shards_per_batch))
    sys.stdout.flush()
def _process_image_files ( thread_index , ranges , name , images , decoder , vocab , num_shards ) : # Each thread produces N shards where N = num_shards / num_threads. For # instance, if num_shards = 128, and num_threads = 2, then the first thread # would produce shards [0, 64). num_threads = len ( ranges ) assert not num_shards % num_threads num_shards_per_batch = int ( num_shards / num_threads ) shard_ranges = np . linspace ( ranges [ thread_index ] [ 0 ] , ranges [ thread_index ] [ 1 ] , num_shards_per_batch + 1 ) . astype ( int ) num_images_in_thread = ranges [ thread_index ] [ 1 ] - ranges [ thread_index ] [ 0 ] counter = 0 for s in range ( num_shards_per_batch ) : # Generate a sharded version of the file name, e.g. 'train-00002-of-00010' shard = thread_index * num_shards_per_batch + s output_filename = "%s-%.5d-of-%.5d" % ( name , shard , num_shards ) output_file = os . path . join ( FLAGS . output_dir , output_filename ) writer = tf . python_io . TFRecordWriter ( output_file ) shard_counter = 0 images_in_shard = np . arange ( shard_ranges [ s ] , shard_ranges [ s + 1 ] , dtype = int ) for i in images_in_shard : image = images [ i ] sequence_example = _to_sequence_example ( image , decoder , vocab ) if sequence_example is not None : writer . write ( sequence_example . SerializeToString ( ) ) shard_counter += 1 counter += 1 if not counter % 1000 : print ( "%s [thread %d]: Processed %d of %d items in thread batch." % ( datetime . now ( ) , thread_index , counter , num_images_in_thread ) ) sys . stdout . flush ( ) writer . close ( ) print ( "%s [thread %d]: Wrote %d image-caption pairs to %s" % ( datetime . now ( ) , thread_index , shard_counter , output_file ) ) sys . stdout . flush ( ) shard_counter = 0 print ( "%s [thread %d]: Wrote %d image-caption pairs to %d shards." % ( datetime . now ( ) , thread_index , counter , num_shards_per_batch ) ) sys . stdout . flush ( )