(self)
Implements DatasetConverter.create_dataset_specification_and_records.
Implements DatasetConverter.create_dataset_specification_and_records.
Implements DatasetConverter . create_dataset_specification_and_records .
def create_dataset_specification_and_records(self):
    """Implements DatasetConverter.create_dataset_specification_and_records."""

    splits = self.get_splits()
    # Get the names of the classes assigned to each split
    train_classes = splits['train']
    valid_classes = splits['valid']
    test_classes = splits['test']

    self.classes_per_split[learning_spec.Split.TRAIN] = len(train_classes)
    self.classes_per_split[learning_spec.Split.VALID] = len(valid_classes)
    self.classes_per_split[learning_spec.Split.TEST] = len(test_classes)

    for classes, split in zip([train_classes, valid_classes, test_classes],
                              ['train', 'val', 'test']):
      path = os.path.join(self.data_root,
                          'mini-imagenet-cache-{}.pkl'.format(split))
      with tf.io.gfile.GFile(path, 'rb') as f:
        data = pkl.load(f)
      # We sort class names to make the dataset creation deterministic
      names = sorted(data['class_dict'].keys())
      for class_id, class_name in zip(classes, names):
        logging.info('Creating record class %d', class_id)
        class_records_path = os.path.join(self.records_path,
                                          self.file_pattern.format(class_id))
        self.class_names[class_id] = class_name
        indices = data['class_dict'][class_name]
        self.images_per_class[class_id] = len(indices)

        writer = tf.python_io.TFRecordWriter(class_records_path)
        for image in data['image_data'][indices]:
          img = Image.fromarray(image)
          buf = io.BytesIO()
          img.save(buf, format='JPEG')
          buf.seek(0)
          write_example(buf.getvalue(), class_id, writer)
        writer.close()
def create_dataset_specification_and_records ( self ) : splits = self . get_splits ( ) # Get the names of the classes assigned to each split train_classes = splits [ 'train' ] valid_classes = splits [ 'valid' ] test_classes = splits [ 'test' ] self . classes_per_split [ learning_spec . Split . TRAIN ] = len ( train_classes ) self . classes_per_split [ learning_spec . Split . VALID ] = len ( valid_classes ) self . classes_per_split [ learning_spec . Split . TEST ] = len ( test_classes ) for classes , split in zip ( [ train_classes , valid_classes , test_classes ] , [ 'train' , 'val' , 'test' ] ) : path = os . path . join ( self . data_root , 'mini-imagenet-cache-{}.pkl' . format ( split ) ) with tf . io . gfile . GFile ( path , 'rb' ) as f : data = pkl . load ( f ) # We sort class names to make the dataset creation deterministic names = sorted ( data [ 'class_dict' ] . keys ( ) ) for class_id , class_name in zip ( classes , names ) : logging . info ( 'Creating record class %d' , class_id ) class_records_path = os . path . join ( self . records_path , self . file_pattern . format ( class_id ) ) self . class_names [ class_id ] = class_name indices = data [ 'class_dict' ] [ class_name ] self . images_per_class [ class_id ] = len ( indices ) writer = tf . python_io . TFRecordWriter ( class_records_path ) for image in data [ 'image_data' ] [ indices ] : img = Image . fromarray ( image ) buf = io . BytesIO ( ) img . save ( buf , format = 'JPEG' ) buf . seek ( 0 ) write_example ( buf . getvalue ( ) , class_id , writer ) writer . close ( )