(dataset_spec_list,
                                      use_dag_ontology_list,
                                      use_bilevel_ontology_list,
                                      split,
                                      episode_descr_config,
                                      pool=None,
                                      shuffle_buffer_size=None,
                                      read_buffer_size_bytes=None,
                                      num_prefetch=0,
                                      image_size=None,
                                      num_to_take=None,
                                      source_sampling_seed=None,
                                      simclr_episode_fraction=0.0)
return dataset
Returns a pipeline emitting data from multiple sources as Episodes.

  Each episode only contains data from one single source. For each episode, its
  source is sampled uniformly across all sources.

  Args:
    dataset_spec_list: A list of DatasetSpecification, one for each source.
    use_dag_ontology_list: A list of Booleans, one for each source: whether to
      use that source's DAG-structured ontology to sample episode classes.
    use_bilevel_ontology_list: A list of Booleans, one for each source: whether
      to use that source's bi-level ontology to sample episode classes.
    split: A learning_spec.Split object identifying the sources split. It is the
      same for all datasets.
    episode_descr_config: An instance of EpisodeDescriptionConfig containing
      parameters relating to sampling shots and ways for episodes.
    pool: String (optional), for example-split datasets, which example split to
      use ('train', 'valid', or 'test'), used at meta-test time only.
    shuffle_buffer_size: int or None, shuffle buffer size for each Dataset.
    read_buffer_size_bytes: int or None, buffer size for each TFRecordDataset.
    num_prefetch: int, the number of examples to prefetch for each class of each
      dataset. Prefetching occurs just after the class-specific Dataset object
      is constructed. If < 1, no prefetching occurs.
    image_size: int, desired image size used during decoding.
    num_to_take: Optional, a list specifying for each dataset the number of
      examples per class to restrict to (for this given split). If provided, its
      length must be the same as len(dataset_spec). If None, no restrictions are
      applied to any dataset and all data per class is used.
    source_sampling_seed: random seed for source sampling.
    simclr_episode_fraction: Float, fraction of episodes that will be converted
      to SimCLR Episodes as described in the CrossTransformers paper.

  Returns:
    A Dataset instance that outputs tuples of fully-assembled and decoded
      episodes zipped with the ID of their data source of origin.
Returns a pipeline emitting data from multiple sources as Episodes.
Returns a pipeline emitting data from multiple sources as Episodes .
def make_multisource_episode_pipeline(dataset_spec_list,
                                      use_dag_ontology_list,
                                      use_bilevel_ontology_list,
                                      split,
                                      episode_descr_config,
                                      pool=None,
                                      shuffle_buffer_size=None,
                                      read_buffer_size_bytes=None,
                                      num_prefetch=0,
                                      image_size=None,
                                      num_to_take=None,
                                      source_sampling_seed=None,
                                      simclr_episode_fraction=0.0):
  """Returns a pipeline emitting data from multiple sources as Episodes.

  Each episode only contains data from one single source. For each episode, its
  source is sampled uniformly across all sources.

  Args:
    dataset_spec_list: A list of DatasetSpecification, one for each source.
    use_dag_ontology_list: A list of Booleans, one for each source: whether to
      use that source's DAG-structured ontology to sample episode classes.
    use_bilevel_ontology_list: A list of Booleans, one for each source: whether
      to use that source's bi-level ontology to sample episode classes.
    split: A learning_spec.Split object identifying the sources split. It is the
      same for all datasets.
    episode_descr_config: An instance of EpisodeDescriptionConfig containing
      parameters relating to sampling shots and ways for episodes.
    pool: String (optional), for example-split datasets, which example split to
      use ('train', 'valid', or 'test'), used at meta-test time only.
    shuffle_buffer_size: int or None, shuffle buffer size for each Dataset.
    read_buffer_size_bytes: int or None, buffer size for each TFRecordDataset.
    num_prefetch: int, the number of examples to prefetch for each class of each
      dataset. Prefetching occurs just after the class-specific Dataset object
      is constructed. If < 1, no prefetching occurs.
    image_size: int, desired image size used during decoding.
    num_to_take: Optional, a list specifying for each dataset the number of
      examples per class to restrict to (for this given split). If provided, its
      length must be the same as len(dataset_spec). If None, no restrictions are
      applied to any dataset and all data per class is used.
    source_sampling_seed: random seed for source sampling.
    simclr_episode_fraction: Float, fraction of episodes that will be converted
      to SimCLR Episodes as described in the CrossTransformers paper.

  Returns:
    A Dataset instance that outputs tuples of fully-assembled and decoded
      episodes zipped with the ID of their data source of origin.
  """
  if pool is not None:
    if not data.POOL_SUPPORTED:
      raise NotImplementedError('Example-level splits or pools not supported.')
  if num_to_take is not None and len(num_to_take) != len(dataset_spec_list):
    raise ValueError('num_to_take does not have the same length as '
                     'dataset_spec_list.')
  if num_to_take is None:
    num_to_take = [-1] * len(dataset_spec_list)
  num_unique_descriptions = episode_descr_config.num_unique_descriptions
  sources = []
  for source_id, (dataset_spec, use_dag_ontology, use_bilevel_ontology,
                  num_to_take_for_dataset) in enumerate(
                      zip(dataset_spec_list, use_dag_ontology_list,
                          use_bilevel_ontology_list, num_to_take)):
    episode_reader = reader.EpisodeReader(dataset_spec, split,
                                          shuffle_buffer_size,
                                          read_buffer_size_bytes, num_prefetch,
                                          num_to_take_for_dataset,
                                          num_unique_descriptions)
    sampler = sampling.EpisodeDescriptionSampler(
        episode_reader.dataset_spec,
        split,
        episode_descr_config,
        pool=pool,
        use_dag_hierarchy=use_dag_ontology,
        use_bilevel_hierarchy=use_bilevel_ontology)
    dataset = episode_reader.create_dataset_input_pipeline(sampler, pool=pool)
    # Create a dataset to zip with the above for identifying the source.
    source_id_dataset = tf.data.Dataset.from_tensors(source_id).repeat()
    sources.append(tf.data.Dataset.zip((dataset, source_id_dataset)))

  # Sample uniformly among sources.
  dataset = tf.data.experimental.sample_from_datasets(
      sources, seed=source_sampling_seed)

  # Episodes coming out of `dataset` contain flushed examples and are internally
  # padded with placeholder examples. `process_episode` discards
  # flushed examples, splits the episode into support and query sets, removes
  # the placeholder examples and decodes the example strings.
  chunk_sizes = sampler.compute_chunk_sizes()

  def map_fn(episode, source_id):
    return process_episode(
        *episode,
        chunk_sizes=chunk_sizes,
        image_size=image_size,
        simclr_episode_fraction=simclr_episode_fraction), source_id

  dataset = dataset.map(map_fn)

  # Overlap episode processing and training.
  dataset = dataset.prefetch(1)
  return dataset
def make_multisource_episode_pipeline ( dataset_spec_list , use_dag_ontology_list , use_bilevel_ontology_list , split , episode_descr_config , pool = None , shuffle_buffer_size = None , read_buffer_size_bytes = None , num_prefetch = 0 , image_size = None , num_to_take = None , source_sampling_seed = None , simclr_episode_fraction = 0.0 ) : if pool is not None : if not data . POOL_SUPPORTED : raise NotImplementedError ( 'Example-level splits or pools not supported.' ) if num_to_take is not None and len ( num_to_take ) != len ( dataset_spec_list ) : raise ValueError ( 'num_to_take does not have the same length as ' 'dataset_spec_list.' ) if num_to_take is None : num_to_take = [ - 1 ] * len ( dataset_spec_list ) num_unique_descriptions = episode_descr_config . num_unique_descriptions sources = [ ] for source_id , ( dataset_spec , use_dag_ontology , use_bilevel_ontology , num_to_take_for_dataset ) in enumerate ( zip ( dataset_spec_list , use_dag_ontology_list , use_bilevel_ontology_list , num_to_take ) ) : episode_reader = reader . EpisodeReader ( dataset_spec , split , shuffle_buffer_size , read_buffer_size_bytes , num_prefetch , num_to_take_for_dataset , num_unique_descriptions ) sampler = sampling . EpisodeDescriptionSampler ( episode_reader . dataset_spec , split , episode_descr_config , pool = pool , use_dag_hierarchy = use_dag_ontology , use_bilevel_hierarchy = use_bilevel_ontology ) dataset = episode_reader . create_dataset_input_pipeline ( sampler , pool = pool ) # Create a dataset to zip with the above for identifying the source. source_id_dataset = tf . data . Dataset . from_tensors ( source_id ) . repeat ( ) sources . append ( tf . data . Dataset . zip ( ( dataset , source_id_dataset ) ) ) # Sample uniformly among sources. dataset = tf . data . experimental . sample_from_datasets ( sources , seed = source_sampling_seed ) # Episodes coming out of `dataset` contain flushed examples and are internally # padded with placeholder examples. `process_episode` discards # flushed examples, splits the episode into support and query sets, removes # the placeholder examples and decodes the example strings. chunk_sizes = sampler . compute_chunk_sizes ( ) def map_fn ( episode , source_id ) : return process_episode ( * episode , chunk_sizes = chunk_sizes , image_size = image_size , simclr_episode_fraction = simclr_episode_fraction ) , source_id dataset = dataset . map ( map_fn ) # Overlap episode processing and training. dataset = dataset . prefetch ( 1 ) return dataset