(metadata, paths)
Generates ImageNet examples.
Generates ImageNet examples.
Generates ImageNet examples .
def generate_ilsvrc_2012_examples(metadata, paths):
  """Generates ImageNet examples."""
  # Enumerate all files to skip.
  files_to_skip = set()
  for other_dataset in ('Caltech101', 'Caltech256', 'CUBirds'):
    lines = paths[f'{other_dataset}_duplicates'].read_text().splitlines()

    for l in lines:
      # Skip comment lines
      l = l.strip()
      if l.startswith('#'):
        continue
      # Lines look like:
      # 'synset/synset_imgnumber.JPEG  # original_file_name.jpg\n'.
      # Extract only the 'synset_imgnumber.JPG' part.
      file_path = l.split('#')[0].strip()
      file_name = os.path.basename(file_path)
      files_to_skip.add(file_name)

  name_to_id = {name: id for id, name in enumerate(metadata['class_names'])}

  for archive_name, archive in  tfds.download.iter_archive(
      paths['ILSVRC2012_img_train'], tfds.download.ExtractMethod.TAR_STREAM):

    class_name = archive_name.split('.')[0]
    label = name_to_id[class_name]
    image_ids = list(range(*metadata['class_slices'][label]))

    archive_iterator = tfds.download.iter_archive(
        archive, tfds.download.ExtractMethod.TAR_STREAM)  # pytype: disable=wrong-arg-types  # gen-stub-imports
    filenames_and_images = sorted([
        (filename, extracted_image.read())
        for filename, extracted_image in archive_iterator
        if filename not in files_to_skip
    ])

    assert len(filenames_and_images) == len(image_ids)

    for image_id, (filename, image_bytes) in zip(image_ids,
                                                 filenames_and_images):
      if filename in files_to_skip:
        logging.info('Skipping file %s', filename)
        continue
      yield _image_key(image_id, metadata['total_num_examples']), {
          'image': _load_and_process_image(image_bytes=image_bytes),
          'format': 'JPEG',
          'filename': filename,
          'label': label,
          'class_name': class_name,
      }
def generate_ilsvrc_2012_examples ( metadata , paths ) : # Enumerate all files to skip. files_to_skip = set ( ) for other_dataset in ( 'Caltech101' , 'Caltech256' , 'CUBirds' ) : lines = paths [ f'{other_dataset}_duplicates' ] . read_text ( ) . splitlines ( ) for l in lines : # Skip comment lines l = l . strip ( ) if l . startswith ( '#' ) : continue # Lines look like: # 'synset/synset_imgnumber.JPEG  # original_file_name.jpg\n'. # Extract only the 'synset_imgnumber.JPG' part. file_path = l . split ( '#' ) [ 0 ] . strip ( ) file_name = os . path . basename ( file_path ) files_to_skip . add ( file_name ) name_to_id = { name : id for id , name in enumerate ( metadata [ 'class_names' ] ) } for archive_name , archive in tfds . download . iter_archive ( paths [ 'ILSVRC2012_img_train' ] , tfds . download . ExtractMethod . TAR_STREAM ) : class_name = archive_name . split ( '.' ) [ 0 ] label = name_to_id [ class_name ] image_ids = list ( range ( * metadata [ 'class_slices' ] [ label ] ) ) archive_iterator = tfds . download . iter_archive ( archive , tfds . download . ExtractMethod . TAR_STREAM ) # pytype: disable=wrong-arg-types  # gen-stub-imports filenames_and_images = sorted ( [ ( filename , extracted_image . read ( ) ) for filename , extracted_image in archive_iterator if filename not in files_to_skip ] ) assert len ( filenames_and_images ) == len ( image_ids ) for image_id , ( filename , image_bytes ) in zip ( image_ids , filenames_and_images ) : if filename in files_to_skip : logging . info ( 'Skipping file %s' , filename ) continue yield _image_key ( image_id , metadata [ 'total_num_examples' ] ) , { 'image' : _load_and_process_image ( image_bytes = image_bytes ) , 'format' : 'JPEG' , 'filename' : filename , 'label' : label , 'class_name' : class_name , }