(self,
                                    batch_size,
                                    offset=0,
                                    pool=None,
                                    shuffle_seed=None)
return dataset
Creates a Dataset encapsulating the input pipeline for one data source.

    Args:
      batch_size: An int representing the max number of examples in each batch.
      offset: An int, that is added to the value of all the targets. This makes
        it possible to have a unique range of targets for each dataset.
      pool: A string (optional) indicating whether to only read examples from a
        given example-level split. If it is provided, these examples will be
        used as 'real test data', and used once each for evaluation only. The
        accepted values are 'valid' and 'test'.
      shuffle_seed: Optional, an int containing the seed passed to
        tf.data.Dataset.shuffle.

    Returns:
      dataset: a tf.data.Dataset instance which encapsulates batch creation for
        the data identified by `dataset_spec` and `split`. These batches contain
        compressed image representations and (possibly offset) absolute class
        IDs. A later part of the pipeline, shared across all sources, will
        decode the example strings.

    Raises:
      ValueError: Invalid pool provided. The supported values are 'valid' and
        'test'.
Creates a Dataset encapsulating the input pipeline for one data source.
Creates a Dataset encapsulating the input pipeline for one data source .
def create_dataset_input_pipeline(self,
                                    batch_size,
                                    offset=0,
                                    pool=None,
                                    shuffle_seed=None):
    """Creates a Dataset encapsulating the input pipeline for one data source.

    Args:
      batch_size: An int representing the max number of examples in each batch.
      offset: An int, that is added to the value of all the targets. This makes
        it possible to have a unique range of targets for each dataset.
      pool: A string (optional) indicating whether to only read examples from a
        given example-level split. If it is provided, these examples will be
        used as 'real test data', and used once each for evaluation only. The
        accepted values are 'valid' and 'test'.
      shuffle_seed: Optional, an int containing the seed passed to
        tf.data.Dataset.shuffle.

    Returns:
      dataset: a tf.data.Dataset instance which encapsulates batch creation for
        the data identified by `dataset_spec` and `split`. These batches contain
        compressed image representations and (possibly offset) absolute class
        IDs. A later part of the pipeline, shared across all sources, will
        decode the example strings.

    Raises:
      ValueError: Invalid pool provided. The supported values are 'valid' and
        'test'.
    """
    if pool and pool not in ['valid', 'test']:
      raise ValueError('Invalid pool provided. The supported values '
                       'are "valid" and "test".')
    # Do not shuffle or repeat each class dataset, to avoid fuzzing epoch
    # boundaries.
    class_datasets = self.construct_class_datasets(
        pool=pool, repeat=False, shuffle=False)
    num_classes = len(class_datasets)

    if pool:
      if not data.POOL_SUPPORTED:
        raise NotImplementedError(
            'Example-level splits or pools not supported.')
    else:
      # To have labels start at 0 and be contiguous, subtracting the starting
      # index from all
      start_ind = self.class_set[0]
      class_set = [
          self.class_set[ds_id] - start_ind for ds_id in range(num_classes)
      ]
      if list(class_set) != list(range(num_classes)):
        raise NotImplementedError('Batch training currently assumes the class '
                                  'set is contiguous and starts at 0.')

      # Sample from each class dataset according to its proportion of examples,
      # so examples from one class should be spread across the whole epoch.
      # Then, shuffle and repeat the combined dataset.
      num_examples_per_class = [
          self.dataset_spec.get_total_images_per_class(class_id, pool=pool)
          for class_id in class_set
      ]
      num_examples_per_class = np.array(num_examples_per_class, 'float64')
      class_proportions = num_examples_per_class / num_examples_per_class.sum()

      # Explicitly skip datasets with a weight of 0, as sample_from_datasets
      # can have some trouble with them.
      new_datasets_and_weights = [
          (dataset, weight)
          for (dataset, weight) in zip(class_datasets, class_proportions)
          if weight > 0
      ]
      class_datasets, class_proportions = zip(*new_datasets_and_weights)
      dataset = tf.data.experimental.sample_from_datasets(
          class_datasets, weights=class_proportions, seed=shuffle_seed)
      if self.shuffle_buffer_size and self.shuffle_buffer_size > 0:
        dataset = dataset.shuffle(
            buffer_size=self.shuffle_buffer_size,
            seed=shuffle_seed,
            reshuffle_each_iteration=True)

    # Using drop_remainder=False for two reasons:
    # - Most importantly, during established splits evaluation, we need to
    #   evaluate on all examples.
    # - Also during training, if the shuffle buffer does not hold all the data,
    #   the last examples are more likely to be dropped than the first ones.
    # In any case, we are handling variable-sized batches just fine, so there
    # is no real reason to drop data.
    dataset = dataset.batch(batch_size, drop_remainder=False)
    if not pool:
      dataset = dataset.repeat()

    if offset:
      map_fn = functools.partial(add_offset_to_target, offset=offset)
      dataset = dataset.map(map_fn)

    # Overlap batching and episode processing.
    dataset = dataset.prefetch(1)

    return dataset
def create_dataset_input_pipeline ( self , batch_size , offset = 0 , pool = None , shuffle_seed = None ) : if pool and pool not in [ 'valid' , 'test' ] : raise ValueError ( 'Invalid pool provided. The supported values ' 'are "valid" and "test".' ) # Do not shuffle or repeat each class dataset, to avoid fuzzing epoch # boundaries. class_datasets = self . construct_class_datasets ( pool = pool , repeat = False , shuffle = False ) num_classes = len ( class_datasets ) if pool : if not data . POOL_SUPPORTED : raise NotImplementedError ( 'Example-level splits or pools not supported.' ) else : # To have labels start at 0 and be contiguous, subtracting the starting # index from all start_ind = self . class_set [ 0 ] class_set = [ self . class_set [ ds_id ] - start_ind for ds_id in range ( num_classes ) ] if list ( class_set ) != list ( range ( num_classes ) ) : raise NotImplementedError ( 'Batch training currently assumes the class ' 'set is contiguous and starts at 0.' ) # Sample from each class dataset according to its proportion of examples, # so examples from one class should be spread across the whole epoch. # Then, shuffle and repeat the combined dataset. num_examples_per_class = [ self . dataset_spec . get_total_images_per_class ( class_id , pool = pool ) for class_id in class_set ] num_examples_per_class = np . array ( num_examples_per_class , 'float64' ) class_proportions = num_examples_per_class / num_examples_per_class . sum ( ) # Explicitly skip datasets with a weight of 0, as sample_from_datasets # can have some trouble with them. new_datasets_and_weights = [ ( dataset , weight ) for ( dataset , weight ) in zip ( class_datasets , class_proportions ) if weight > 0 ] class_datasets , class_proportions = zip ( * new_datasets_and_weights ) dataset = tf . data . experimental . sample_from_datasets ( class_datasets , weights = class_proportions , seed = shuffle_seed ) if self . shuffle_buffer_size and self . shuffle_buffer_size > 0 : dataset = dataset . shuffle ( buffer_size = self . shuffle_buffer_size , seed = shuffle_seed , reshuffle_each_iteration = True ) # Using drop_remainder=False for two reasons: # - Most importantly, during established splits evaluation, we need to #   evaluate on all examples. # - Also during training, if the shuffle buffer does not hold all the data, #   the last examples are more likely to be dropped than the first ones. # In any case, we are handling variable-sized batches just fine, so there # is no real reason to drop data. dataset = dataset . batch ( batch_size , drop_remainder = False ) if not pool : dataset = dataset . repeat ( ) if offset : map_fn = functools . partial ( add_offset_to_target , offset = offset ) dataset = dataset . map ( map_fn ) # Overlap batching and episode processing. dataset = dataset . prefetch ( 1 ) return dataset