(self,
                                    sampler,
                                    pool=None,
                                    shuffle_seed=None)
return dataset
Creates a Dataset encapsulating the input pipeline for one data source.

    Args:
      sampler: EpisodeDescriptionSampler instance.
      pool: A string (optional) indicating whether to only read examples from a
        given example-level split.
      shuffle_seed: Optional, an int containing the seed passed to
        tf.data.Dataset.shuffle.

    Returns:
      dataset: a tf.data.Dataset instance which encapsulates episode creation
        for the data identified by `dataset_spec` and `split`. These episodes
        contain flushed examples and are internally padded with placeholders.
        A later part of the pipeline, shared across all sources, will extract
        support and query sets and decode the example strings.
Creates a Dataset encapsulating the input pipeline for one data source.
Creates a Dataset encapsulating the input pipeline for one data source .
def create_dataset_input_pipeline(self,
                                    sampler,
                                    pool=None,
                                    shuffle_seed=None):
    """Creates a Dataset encapsulating the input pipeline for one data source.

    Args:
      sampler: EpisodeDescriptionSampler instance.
      pool: A string (optional) indicating whether to only read examples from a
        given example-level split.
      shuffle_seed: Optional, an int containing the seed passed to
        tf.data.Dataset.shuffle.

    Returns:
      dataset: a tf.data.Dataset instance which encapsulates episode creation
        for the data identified by `dataset_spec` and `split`. These episodes
        contain flushed examples and are internally padded with placeholders.
        A later part of the pipeline, shared across all sources, will extract
        support and query sets and decode the example strings.
    """
    # Always shuffle, unless self.shuffle_buffer_size is 0
    shuffle = (self.shuffle_buffer_size and self.shuffle_buffer_size > 0)
    class_datasets = self.construct_class_datasets(
        pool=pool, shuffle=shuffle, shuffle_seed=shuffle_seed)

    # We also construct a placeholder dataset which outputs
    # `(b'', PLACEHOLDER_CLASS_ID)` tuples.
    placeholder_dataset = tf.data.Dataset.zip(
        (tf.data.Dataset.from_tensors(b'').repeat(),
         tf.data.Dataset.from_tensors(PLACEHOLDER_CLASS_ID).repeat()))
    class_datasets.append(placeholder_dataset)

    # The "choice" dataset outputs a stream of dataset IDs which are used to
    # select which class dataset to sample from. We turn the stream of dataset
    # IDs into a stream of `(example_string, class_id)` tuples using
    # `choose_from_datasets`.
    representation_generator = functools.partial(
        episode_representation_generator,
        dataset_spec=self.dataset_spec,
        split=self.split,
        pool=pool,
        sampler=sampler)

    if not self.num_unique_descriptions:
      choice_dataset = tf.data.Dataset.from_generator(representation_generator,
                                                      (tf.int64),
                                                      tf.TensorShape([None, 2]))
    else:
      # If num_unique_descriptions is x > 0, then we pre-generate x number of
      # episodes and repeatedly iterate over them.
      representations = list(
          map(
              # We need to use an intermediate string representation in order to
              # shuffle ragged arrays with tf.data.Dataset.
              tf.io.serialize_tensor,
              itertools.islice(representation_generator(),
                               self.num_unique_descriptions)))
      choice_dataset = tf.data.Dataset.from_tensor_slices(
          representations).shuffle(self.num_unique_descriptions).map(
              lambda s: tf.io.parse_tensor(s, tf.int64)).repeat()

    choice_dataset = choice_dataset.map(
        decompress_episode_representation).unbatch()

    dataset = tf.data.experimental.choose_from_datasets(class_datasets,
                                                        choice_dataset)

    # Episodes have a fixed size prescribed by `sampler.compute_chunk_sizes`.
    dataset = dataset.batch(sum(sampler.compute_chunk_sizes()))
    # Overlap batching and episode processing.
    dataset = dataset.prefetch(1)

    return dataset
def create_dataset_input_pipeline ( self , sampler , pool = None , shuffle_seed = None ) : # Always shuffle, unless self.shuffle_buffer_size is 0 shuffle = ( self . shuffle_buffer_size and self . shuffle_buffer_size > 0 ) class_datasets = self . construct_class_datasets ( pool = pool , shuffle = shuffle , shuffle_seed = shuffle_seed ) # We also construct a placeholder dataset which outputs # `(b'', PLACEHOLDER_CLASS_ID)` tuples. placeholder_dataset = tf . data . Dataset . zip ( ( tf . data . Dataset . from_tensors ( b'' ) . repeat ( ) , tf . data . Dataset . from_tensors ( PLACEHOLDER_CLASS_ID ) . repeat ( ) ) ) class_datasets . append ( placeholder_dataset ) # The "choice" dataset outputs a stream of dataset IDs which are used to # select which class dataset to sample from. We turn the stream of dataset # IDs into a stream of `(example_string, class_id)` tuples using # `choose_from_datasets`. representation_generator = functools . partial ( episode_representation_generator , dataset_spec = self . dataset_spec , split = self . split , pool = pool , sampler = sampler ) if not self . num_unique_descriptions : choice_dataset = tf . data . Dataset . from_generator ( representation_generator , ( tf . int64 ) , tf . TensorShape ( [ None , 2 ] ) ) else : # If num_unique_descriptions is x > 0, then we pre-generate x number of # episodes and repeatedly iterate over them. representations = list ( map ( # We need to use an intermediate string representation in order to # shuffle ragged arrays with tf.data.Dataset. tf . io . serialize_tensor , itertools . islice ( representation_generator ( ) , self . num_unique_descriptions ) ) ) choice_dataset = tf . data . Dataset . from_tensor_slices ( representations ) . shuffle ( self . num_unique_descriptions ) . map ( lambda s : tf . io . parse_tensor ( s , tf . int64 ) ) . repeat ( ) choice_dataset = choice_dataset . map ( decompress_episode_representation ) . unbatch ( ) dataset = tf . data . experimental . choose_from_datasets ( class_datasets , choice_dataset ) # Episodes have a fixed size prescribed by `sampler.compute_chunk_sizes`. dataset = dataset . batch ( sum ( sampler . compute_chunk_sizes ( ) ) ) # Overlap batching and episode processing. dataset = dataset . prefetch ( 1 ) return dataset