(self)
Implements DatasetConverter.create_dataset_specification_and_records.
Implements DatasetConverter.create_dataset_specification_and_records.
Implements DatasetConverter . create_dataset_specification_and_records .
def create_dataset_specification_and_records(self):
    """Implements DatasetConverter.create_dataset_specification_and_records."""

    splits = self.get_splits()
    # Get the names of the classes assigned to each split
    train_classes = splits['train']
    valid_classes = splits['valid']
    test_classes = splits['test']

    self.classes_per_split[learning_spec.Split.TRAIN] = len(train_classes)
    self.classes_per_split[learning_spec.Split.VALID] = len(valid_classes)
    self.classes_per_split[learning_spec.Split.TEST] = len(test_classes)

    # Retrieve mapping from filename to bounding box.
    # Cropping to the bounding boxes is important for two reasons:
    # 1) The dataset documentation mentions that "[the] (main) aircraft in each
    #    image is annotated with a tight bounding box [...]", which suggests
    #    that there may be more than one aircraft in some images. Cropping to
    #    the bounding boxes removes ambiguity as to which airplane the label
    #    refers to.
    # 2) Raw images have a 20-pixel border at the bottom with copyright
    #    information which needs to be removed. Cropping to the bounding boxes
    #    has the side-effect that it removes the border.
    bboxes_path = os.path.join(self.data_root, 'data', 'images_box.txt')
    with tf.io.gfile.GFile(bboxes_path, 'r') as f:
      names_to_bboxes = [
          line.split('\n')[0].split(' ') for line in f.readlines()
      ]
      names_to_bboxes = dict(
          (name, map(int, (xmin, ymin, xmax, ymax)))
          for name, xmin, ymin, xmax, ymax in names_to_bboxes)

    # Retrieve mapping from filename to variant
    variant_trainval_path = os.path.join(self.data_root, 'data',
                                         'images_variant_trainval.txt')
    with tf.io.gfile.GFile(variant_trainval_path, 'r') as f:
      names_to_variants = [
          line.split('\n')[0].split(' ', 1) for line in f.readlines()
      ]

    variant_test_path = os.path.join(self.data_root, 'data',
                                     'images_variant_test.txt')
    with tf.io.gfile.GFile(variant_test_path, 'r') as f:
      names_to_variants += [
          line.split('\n')[0].split(' ', 1) for line in f.readlines()
      ]

    names_to_variants = dict(names_to_variants)

    # Build mapping from variant to filenames. "Variant" refers to the aircraft
    # model variant (e.g., A330-200) and is used as the class name in the
    # dataset. The position of the class name in the concatenated list of
    # training, validation, and test class name constitutes its class ID.
    variants_to_names = collections.defaultdict(list)
    for name, variant in names_to_variants.items():
      variants_to_names[variant].append(name)

    all_classes = list(
        itertools.chain(train_classes, valid_classes, test_classes))
    assert set(variants_to_names.keys()) == set(all_classes)

    for class_id, class_name in enumerate(all_classes):
      logging.info('Creating record for class ID %d (%s)...', class_id,
                   class_name)
      class_files = [
          os.path.join(self.data_root, 'data', 'images',
                       '{}.jpg'.format(filename))
          for filename in sorted(variants_to_names[class_name])
      ]
      bboxes = [
          names_to_bboxes[name]
          for name in sorted(variants_to_names[class_name])
      ]
      class_records_path = os.path.join(
          self.records_path, self.dataset_spec.file_pattern.format(class_id))
      self.class_names[class_id] = class_name
      self.images_per_class[class_id] = len(class_files)

      write_tfrecord_from_image_files(
          class_files, class_id, class_records_path, bboxes=bboxes)
def create_dataset_specification_and_records ( self ) : splits = self . get_splits ( ) # Get the names of the classes assigned to each split train_classes = splits [ 'train' ] valid_classes = splits [ 'valid' ] test_classes = splits [ 'test' ] self . classes_per_split [ learning_spec . Split . TRAIN ] = len ( train_classes ) self . classes_per_split [ learning_spec . Split . VALID ] = len ( valid_classes ) self . classes_per_split [ learning_spec . Split . TEST ] = len ( test_classes ) # Retrieve mapping from filename to bounding box. # Cropping to the bounding boxes is important for two reasons: # 1) The dataset documentation mentions that "[the] (main) aircraft in each #    image is annotated with a tight bounding box [...]", which suggests #    that there may be more than one aircraft in some images. Cropping to #    the bounding boxes removes ambiguity as to which airplane the label #    refers to. # 2) Raw images have a 20-pixel border at the bottom with copyright #    information which needs to be removed. Cropping to the bounding boxes #    has the side-effect that it removes the border. bboxes_path = os . path . join ( self . data_root , 'data' , 'images_box.txt' ) with tf . io . gfile . GFile ( bboxes_path , 'r' ) as f : names_to_bboxes = [ line . split ( '\n' ) [ 0 ] . split ( ' ' ) for line in f . readlines ( ) ] names_to_bboxes = dict ( ( name , map ( int , ( xmin , ymin , xmax , ymax ) ) ) for name , xmin , ymin , xmax , ymax in names_to_bboxes ) # Retrieve mapping from filename to variant variant_trainval_path = os . path . join ( self . data_root , 'data' , 'images_variant_trainval.txt' ) with tf . io . gfile . GFile ( variant_trainval_path , 'r' ) as f : names_to_variants = [ line . split ( '\n' ) [ 0 ] . split ( ' ' , 1 ) for line in f . readlines ( ) ] variant_test_path = os . path . join ( self . data_root , 'data' , 'images_variant_test.txt' ) with tf . io . gfile . GFile ( variant_test_path , 'r' ) as f : names_to_variants += [ line . split ( '\n' ) [ 0 ] . split ( ' ' , 1 ) for line in f . readlines ( ) ] names_to_variants = dict ( names_to_variants ) # Build mapping from variant to filenames. "Variant" refers to the aircraft # model variant (e.g., A330-200) and is used as the class name in the # dataset. The position of the class name in the concatenated list of # training, validation, and test class name constitutes its class ID. variants_to_names = collections . defaultdict ( list ) for name , variant in names_to_variants . items ( ) : variants_to_names [ variant ] . append ( name ) all_classes = list ( itertools . chain ( train_classes , valid_classes , test_classes ) ) assert set ( variants_to_names . keys ( ) ) == set ( all_classes ) for class_id , class_name in enumerate ( all_classes ) : logging . info ( 'Creating record for class ID %d (%s)...' , class_id , class_name ) class_files = [ os . path . join ( self . data_root , 'data' , 'images' , '{}.jpg' . format ( filename ) ) for filename in sorted ( variants_to_names [ class_name ] ) ] bboxes = [ names_to_bboxes [ name ] for name in sorted ( variants_to_names [ class_name ] ) ] class_records_path = os . path . join ( self . records_path , self . dataset_spec . file_pattern . format ( class_id ) ) self . class_names [ class_id ] = class_name self . images_per_class [ class_id ] = len ( class_files ) write_tfrecord_from_image_files ( class_files , class_id , class_records_path , bboxes = bboxes )