(self,
                               pool=None,
                               repeat=True,
                               shuffle=True,
                               shuffle_seed=None)
return class_datasets
Constructs the list of class datasets.

    Args:
      pool: A string (optional) indicating whether to only read examples from a
        given example-level split.
      repeat: Boolean indicating whether each of the class datasets should be
        repeated (to provide an infinite stream) or not.
      shuffle: Boolean indicating whether each of the class datasets should be
        shuffled or not.
      shuffle_seed: Optional, an int containing the seed passed to
        tf.data.Dataset.shuffle.

    Returns:
      class_datasets: list of tf.data.Dataset, one for each class.
Constructs the list of class datasets.
Constructs the list of class datasets .
def construct_class_datasets(self,
                               pool=None,
                               repeat=True,
                               shuffle=True,
                               shuffle_seed=None):
    """Constructs the list of class datasets.

    Args:
      pool: A string (optional) indicating whether to only read examples from a
        given example-level split.
      repeat: Boolean indicating whether each of the class datasets should be
        repeated (to provide an infinite stream) or not.
      shuffle: Boolean indicating whether each of the class datasets should be
        shuffled or not.
      shuffle_seed: Optional, an int containing the seed passed to
        tf.data.Dataset.shuffle.

    Returns:
      class_datasets: list of tf.data.Dataset, one for each class.
    """
    file_pattern = self.dataset_spec.file_pattern
    # We construct one dataset object per class. Each dataset outputs a stream
    # of `(example_string, dataset_id)` tuples.
    class_datasets = []
    for dataset_id in range(self.num_classes):
      class_id = self.class_set[dataset_id]
      if pool:
        if not data.POOL_SUPPORTED:
          raise NotImplementedError(
              'Example-level splits or pools not supported.')
      else:
        if file_pattern.startswith('{}_{}'):
          # TODO(lamblinp): Add support for sharded files if needed.
          raise NotImplementedError('Sharded files are not supported yet. '
                                    'The code expects one dataset per class.')
        elif file_pattern.startswith('{}'):
          filename = os.path.join(self.base_path, file_pattern.format(class_id))
        else:
          raise ValueError('Unsupported file_pattern in DatasetSpec: %s. '
                           'Expected something starting with "{}" or "{}_{}".' %
                           file_pattern)

      example_string_dataset = tf.data.TFRecordDataset(
          filename, buffer_size=self.read_buffer_size_bytes)

      # Create a dataset containing only num_to_take elements from
      # example_string_dataset. By default, takes all elements.
      example_string_dataset = example_string_dataset.take(self.num_to_take)

      if self.num_prefetch > 0:
        example_string_dataset = example_string_dataset.prefetch(
            self.num_prefetch)
      if shuffle:
        # Do not set a buffer size greater than the number of examples in this
        # class, as it can result in unnecessary memory being allocated.
        num_examples = self.dataset_spec.get_total_images_per_class(
            class_id, pool=pool)
        shuffle_buffer_size = min(num_examples, self.shuffle_buffer_size)
        if shuffle_buffer_size > 1:
          example_string_dataset = example_string_dataset.shuffle(
              buffer_size=shuffle_buffer_size,
              seed=shuffle_seed,
              reshuffle_each_iteration=True)
      if repeat:
        example_string_dataset = example_string_dataset.repeat()

      # These are absolute, dataset-specific class IDs (not relative to a given
      # split). It is okay to have class ID collisions across datasets, since we
      # don't sample multi-dataset episodes.
      class_id_dataset = tf.data.Dataset.from_tensors(class_id).repeat()
      dataset = tf.data.Dataset.zip((example_string_dataset, class_id_dataset))
      class_datasets.append(dataset)

    assert len(class_datasets) == self.num_classes
    return class_datasets
def construct_class_datasets ( self , pool = None , repeat = True , shuffle = True , shuffle_seed = None ) : file_pattern = self . dataset_spec . file_pattern # We construct one dataset object per class. Each dataset outputs a stream # of `(example_string, dataset_id)` tuples. class_datasets = [ ] for dataset_id in range ( self . num_classes ) : class_id = self . class_set [ dataset_id ] if pool : if not data . POOL_SUPPORTED : raise NotImplementedError ( 'Example-level splits or pools not supported.' ) else : if file_pattern . startswith ( '{}_{}' ) : # TODO(lamblinp): Add support for sharded files if needed. raise NotImplementedError ( 'Sharded files are not supported yet. ' 'The code expects one dataset per class.' ) elif file_pattern . startswith ( '{}' ) : filename = os . path . join ( self . base_path , file_pattern . format ( class_id ) ) else : raise ValueError ( 'Unsupported file_pattern in DatasetSpec: %s. ' 'Expected something starting with "{}" or "{}_{}".' % file_pattern ) example_string_dataset = tf . data . TFRecordDataset ( filename , buffer_size = self . read_buffer_size_bytes ) # Create a dataset containing only num_to_take elements from # example_string_dataset. By default, takes all elements. example_string_dataset = example_string_dataset . take ( self . num_to_take ) if self . num_prefetch > 0 : example_string_dataset = example_string_dataset . prefetch ( self . num_prefetch ) if shuffle : # Do not set a buffer size greater than the number of examples in this # class, as it can result in unnecessary memory being allocated. num_examples = self . dataset_spec . get_total_images_per_class ( class_id , pool = pool ) shuffle_buffer_size = min ( num_examples , self . shuffle_buffer_size ) if shuffle_buffer_size > 1 : example_string_dataset = example_string_dataset . shuffle ( buffer_size = shuffle_buffer_size , seed = shuffle_seed , reshuffle_each_iteration = True ) if repeat : example_string_dataset = example_string_dataset . repeat ( ) # These are absolute, dataset-specific class IDs (not relative to a given # split). It is okay to have class ID collisions across datasets, since we # don't sample multi-dataset episodes. class_id_dataset = tf . data . Dataset . from_tensors ( class_id ) . repeat ( ) dataset = tf . data . Dataset . zip ( ( example_string_dataset , class_id_dataset ) ) class_datasets . append ( dataset ) assert len ( class_datasets ) == self . num_classes return class_datasets