(self,
               dataset_spec,
               split,
               episode_descr_config,
               pool=None,
               use_dag_hierarchy=False,
               use_bilevel_hierarchy=False,
               use_all_classes=False,
               ignore_hierarchy_probability=0.0)
Initializes an EpisodeDescriptionSampler.episode_config.

    Args:
      dataset_spec: DatasetSpecification, dataset specification.
      split: one of Split.TRAIN, Split.VALID, or Split.TEST.
      episode_descr_config: An instance of EpisodeDescriptionConfig containing
        parameters relating to sampling shots and ways for episodes.
      pool: A string ('train' or 'test') or None, indicating which example-level
        split to select, if the current dataset has them.
      use_dag_hierarchy: Boolean, defaults to False. If a DAG-structured
        ontology is defined in dataset_spec, use it to choose related classes.
      use_bilevel_hierarchy: Boolean, defaults to False. If a bi-level ontology
        is defined in dataset_spec, use it for sampling classes.
      use_all_classes: Boolean, defaults to False. Uses all available classes,
        in order, instead of sampling. Overrides `num_ways` to the number of
        classes in `split`.
      ignore_hierarchy_probability: Float, if using a hierarchy, this flag makes
        the sampler ignore the hierarchy for this proportion of episodes and
        instead sample categories uniformly.

    Raises:
      RuntimeError: if required parameters are missing.
      ValueError: Inconsistent parameters.
Initializes an EpisodeDescriptionSampler.episode_config.
Initializes an EpisodeDescriptionSampler . episode_config .
def __init__(self,
               dataset_spec,
               split,
               episode_descr_config,
               pool=None,
               use_dag_hierarchy=False,
               use_bilevel_hierarchy=False,
               use_all_classes=False,
               ignore_hierarchy_probability=0.0):
    """Initializes an EpisodeDescriptionSampler.episode_config.

    Args:
      dataset_spec: DatasetSpecification, dataset specification.
      split: one of Split.TRAIN, Split.VALID, or Split.TEST.
      episode_descr_config: An instance of EpisodeDescriptionConfig containing
        parameters relating to sampling shots and ways for episodes.
      pool: A string ('train' or 'test') or None, indicating which example-level
        split to select, if the current dataset has them.
      use_dag_hierarchy: Boolean, defaults to False. If a DAG-structured
        ontology is defined in dataset_spec, use it to choose related classes.
      use_bilevel_hierarchy: Boolean, defaults to False. If a bi-level ontology
        is defined in dataset_spec, use it for sampling classes.
      use_all_classes: Boolean, defaults to False. Uses all available classes,
        in order, instead of sampling. Overrides `num_ways` to the number of
        classes in `split`.
      ignore_hierarchy_probability: Float, if using a hierarchy, this flag makes
        the sampler ignore the hierarchy for this proportion of episodes and
        instead sample categories uniformly.

    Raises:
      RuntimeError: if required parameters are missing.
      ValueError: Inconsistent parameters.
    """
    # Each instance has its own RNG which is seeded from the module-level RNG,
    # which makes episode description sampling deterministic within individual
    # data sources.
    self._rng = np.random.RandomState(
        seed=RNG.randint(0, 2**32, size=None, dtype='uint32'))
    self.dataset_spec = dataset_spec
    self.split = split
    self.pool = pool
    self.use_dag_hierarchy = use_dag_hierarchy
    self.use_bilevel_hierarchy = use_bilevel_hierarchy
    self.ignore_hierarchy_probability = ignore_hierarchy_probability
    self.use_all_classes = use_all_classes
    self.num_ways = episode_descr_config.num_ways
    self.num_support = episode_descr_config.num_support
    self.num_query = episode_descr_config.num_query
    self.min_ways = episode_descr_config.min_ways
    self.max_ways_upper_bound = episode_descr_config.max_ways_upper_bound
    self.max_num_query = episode_descr_config.max_num_query
    self.max_support_set_size = episode_descr_config.max_support_set_size
    self.max_support_size_contrib_per_class = episode_descr_config.max_support_size_contrib_per_class
    self.min_log_weight = episode_descr_config.min_log_weight
    self.max_log_weight = episode_descr_config.max_log_weight
    self.min_examples_in_class = episode_descr_config.min_examples_in_class
    self.episode_description_switch_frequency = episode_descr_config.episode_description_switch_frequency

    self.class_set = dataset_spec.get_classes(self.split)
    self.num_classes = len(self.class_set)
    # Filter out classes with too few examples
    self._filtered_class_set = []
    # Store (class_id, n_examples) of skipped classes for logging.
    skipped_classes = []
    for class_id in self.class_set:
      n_examples = dataset_spec.get_total_images_per_class(class_id, pool=pool)
      if n_examples < self.min_examples_in_class:
        skipped_classes.append((class_id, n_examples))
      else:
        self._filtered_class_set.append(class_id)
    self.num_filtered_classes = len(self._filtered_class_set)

    if skipped_classes:
      logging.info(
          'Skipping the following classes, which do not have at least '
          '%d examples', self.min_examples_in_class)
    for class_id, n_examples in skipped_classes:
      logging.info('%s (ID=%d, %d examples)',
                   dataset_spec.class_names[class_id], class_id, n_examples)

    if self.min_ways and self.num_filtered_classes < self.min_ways:
      raise ValueError(
          '"min_ways" is set to {}, but split {} of dataset {} only has {} '
          'classes with at least {} examples ({} total), so it is not possible '
          'to create an episode for it. This may have resulted from applying a '
          'restriction on this split of this dataset by specifying '
          'benchmark.restrict_classes or benchmark.min_examples_in_class.'
          .format(self.min_ways, split, dataset_spec.name,
                  self.num_filtered_classes, self.min_examples_in_class,
                  self.num_classes))

    if self.use_all_classes:
      if self.num_classes != self.num_filtered_classes:
        raise ValueError('"use_all_classes" is not compatible with a value of '
                         '"min_examples_in_class" ({}) that results in some '
                         'classes being excluded.'.format(
                             self.min_examples_in_class))
      self.num_ways = self.num_classes

    # Maybe overwrite use_dag_hierarchy or use_bilevel_hierarchy if requested.
    if episode_descr_config.ignore_dag_ontology:
      self.use_dag_hierarchy = False
    if episode_descr_config.ignore_bilevel_ontology:
      self.use_bilevel_hierarchy = False

    # For Omniglot.
    if self.use_bilevel_hierarchy:
      if self.num_ways is not None:
        raise ValueError('"use_bilevel_hierarchy" is incompatible with '
                         '"num_ways".')
      if self.min_examples_in_class > 0:
        raise ValueError('"use_bilevel_hierarchy" is incompatible with '
                         '"min_examples_in_class".')
      if self.use_dag_hierarchy:
        raise ValueError('"use_bilevel_hierarchy" is incompatible with '
                         '"use_dag_hierarchy".')

      if not isinstance(dataset_spec,
                        dataset_spec_lib.BiLevelDatasetSpecification):
        raise ValueError('Only applicable to datasets with a bi-level '
                         'dataset specification.')
      # The id's of the superclasses of the split (a contiguous range of ints).
      all_superclasses = dataset_spec.get_superclasses(self.split)
      self.superclass_set = []
      for i in all_superclasses:
        if self.dataset_spec.classes_per_superclass[i] < self.min_ways:
          raise ValueError(
              'Superclass: %d has num_classes=%d < min_ways=%d.' %
              (i, self.dataset_spec.classes_per_superclass[i], self.min_ways))
        self.superclass_set.append(i)
    # For ImageNet.
    elif self.use_dag_hierarchy:
      if self.num_ways is not None:
        raise ValueError('"use_dag_hierarchy" is incompatible with "num_ways".')

      if not isinstance(dataset_spec,
                        dataset_spec_lib.HierarchicalDatasetSpecification):
        raise ValueError('Only applicable to datasets with a hierarchical '
                         'dataset specification.')

      # A DAG for navigating the ontology for the given split.
      graph = dataset_spec.get_split_subgraph(self.split)

      # Map the absolute class IDs in the split's class set to IDs relative to
      # the split.
      class_set = self.class_set
      abs_to_rel_ids = dict((abs_id, i) for i, abs_id in enumerate(class_set))

      # Extract the sets of leaves and internal nodes in the DAG.
      leaves = set(imagenet_specification.get_leaves(graph))
      internal_nodes = graph - leaves  # set difference

      # Map each node of the DAG to the Synsets of the leaves it spans.
      spanning_leaves_dict = imagenet_specification.get_spanning_leaves(graph)

      # Build a list of lists storing the relative class IDs of the spanning
      # leaves for each eligible internal node. We ensure a deterministic order
      # by sorting the inner-nodes and their corresponding leaves by wn_id.
      self.span_leaves_rel = []
      for node in sorted(internal_nodes, key=lambda n: n.wn_id):
        node_leaves = sorted(spanning_leaves_dict[node], key=lambda n: n.wn_id)
        # Build a list of relative class IDs of leaves that have at least
        # min_examples_in_class examples.
        ids_rel = []
        for leaf in node_leaves:
          abs_id = dataset_spec.class_names_to_ids[leaf.wn_id]
          if abs_id in self._filtered_class_set:
            ids_rel.append(abs_to_rel_ids[abs_id])

        # Internal nodes are eligible if they span at least
        # `min_allowed_classes` and at most `max_eligible` leaves.
        if self.min_ways <= len(ids_rel) <= MAX_SPANNING_LEAVES_ELIGIBLE:
          self.span_leaves_rel.append(ids_rel)

      num_eligible_nodes = len(self.span_leaves_rel)
      if num_eligible_nodes < 1:
        raise ValueError('There are no classes eligible for participating in '
                         'episodes. Consider changing the value of '
                         '`EpisodeDescriptionSampler.min_ways` in gin, or '
                         'or MAX_SPANNING_LEAVES_ELIGIBLE in data.py.')
def __init__ ( self , dataset_spec , split , episode_descr_config , pool = None , use_dag_hierarchy = False , use_bilevel_hierarchy = False , use_all_classes = False , ignore_hierarchy_probability = 0.0 ) : # Each instance has its own RNG which is seeded from the module-level RNG, # which makes episode description sampling deterministic within individual # data sources. self . _rng = np . random . RandomState ( seed = RNG . randint ( 0 , 2 ** 32 , size = None , dtype = 'uint32' ) ) self . dataset_spec = dataset_spec self . split = split self . pool = pool self . use_dag_hierarchy = use_dag_hierarchy self . use_bilevel_hierarchy = use_bilevel_hierarchy self . ignore_hierarchy_probability = ignore_hierarchy_probability self . use_all_classes = use_all_classes self . num_ways = episode_descr_config . num_ways self . num_support = episode_descr_config . num_support self . num_query = episode_descr_config . num_query self . min_ways = episode_descr_config . min_ways self . max_ways_upper_bound = episode_descr_config . max_ways_upper_bound self . max_num_query = episode_descr_config . max_num_query self . max_support_set_size = episode_descr_config . max_support_set_size self . max_support_size_contrib_per_class = episode_descr_config . max_support_size_contrib_per_class self . min_log_weight = episode_descr_config . min_log_weight self . max_log_weight = episode_descr_config . max_log_weight self . min_examples_in_class = episode_descr_config . min_examples_in_class self . episode_description_switch_frequency = episode_descr_config . episode_description_switch_frequency self . class_set = dataset_spec . get_classes ( self . split ) self . num_classes = len ( self . class_set ) # Filter out classes with too few examples self . _filtered_class_set = [ ] # Store (class_id, n_examples) of skipped classes for logging. skipped_classes = [ ] for class_id in self . class_set : n_examples = dataset_spec . get_total_images_per_class ( class_id , pool = pool ) if n_examples < self . min_examples_in_class : skipped_classes . append ( ( class_id , n_examples ) ) else : self . _filtered_class_set . append ( class_id ) self . num_filtered_classes = len ( self . _filtered_class_set ) if skipped_classes : logging . info ( 'Skipping the following classes, which do not have at least ' '%d examples' , self . min_examples_in_class ) for class_id , n_examples in skipped_classes : logging . info ( '%s (ID=%d, %d examples)' , dataset_spec . class_names [ class_id ] , class_id , n_examples ) if self . min_ways and self . num_filtered_classes < self . min_ways : raise ValueError ( '"min_ways" is set to {}, but split {} of dataset {} only has {} ' 'classes with at least {} examples ({} total), so it is not possible ' 'to create an episode for it. This may have resulted from applying a ' 'restriction on this split of this dataset by specifying ' 'benchmark.restrict_classes or benchmark.min_examples_in_class.' . format ( self . min_ways , split , dataset_spec . name , self . num_filtered_classes , self . min_examples_in_class , self . num_classes ) ) if self . use_all_classes : if self . num_classes != self . num_filtered_classes : raise ValueError ( '"use_all_classes" is not compatible with a value of ' '"min_examples_in_class" ({}) that results in some ' 'classes being excluded.' . format ( self . min_examples_in_class ) ) self . num_ways = self . num_classes # Maybe overwrite use_dag_hierarchy or use_bilevel_hierarchy if requested. if episode_descr_config . ignore_dag_ontology : self . use_dag_hierarchy = False if episode_descr_config . ignore_bilevel_ontology : self . use_bilevel_hierarchy = False # For Omniglot. if self . use_bilevel_hierarchy : if self . num_ways is not None : raise ValueError ( '"use_bilevel_hierarchy" is incompatible with ' '"num_ways".' ) if self . min_examples_in_class > 0 : raise ValueError ( '"use_bilevel_hierarchy" is incompatible with ' '"min_examples_in_class".' ) if self . use_dag_hierarchy : raise ValueError ( '"use_bilevel_hierarchy" is incompatible with ' '"use_dag_hierarchy".' ) if not isinstance ( dataset_spec , dataset_spec_lib . BiLevelDatasetSpecification ) : raise ValueError ( 'Only applicable to datasets with a bi-level ' 'dataset specification.' ) # The id's of the superclasses of the split (a contiguous range of ints). all_superclasses = dataset_spec . get_superclasses ( self . split ) self . superclass_set = [ ] for i in all_superclasses : if self . dataset_spec . classes_per_superclass [ i ] < self . min_ways : raise ValueError ( 'Superclass: %d has num_classes=%d < min_ways=%d.' % ( i , self . dataset_spec . classes_per_superclass [ i ] , self . min_ways ) ) self . superclass_set . append ( i ) # For ImageNet. elif self . use_dag_hierarchy : if self . num_ways is not None : raise ValueError ( '"use_dag_hierarchy" is incompatible with "num_ways".' ) if not isinstance ( dataset_spec , dataset_spec_lib . HierarchicalDatasetSpecification ) : raise ValueError ( 'Only applicable to datasets with a hierarchical ' 'dataset specification.' ) # A DAG for navigating the ontology for the given split. graph = dataset_spec . get_split_subgraph ( self . split ) # Map the absolute class IDs in the split's class set to IDs relative to # the split. class_set = self . class_set abs_to_rel_ids = dict ( ( abs_id , i ) for i , abs_id in enumerate ( class_set ) ) # Extract the sets of leaves and internal nodes in the DAG. leaves = set ( imagenet_specification . get_leaves ( graph ) ) internal_nodes = graph - leaves # set difference # Map each node of the DAG to the Synsets of the leaves it spans. spanning_leaves_dict = imagenet_specification . get_spanning_leaves ( graph ) # Build a list of lists storing the relative class IDs of the spanning # leaves for each eligible internal node. We ensure a deterministic order # by sorting the inner-nodes and their corresponding leaves by wn_id. self . span_leaves_rel = [ ] for node in sorted ( internal_nodes , key = lambda n : n . wn_id ) : node_leaves = sorted ( spanning_leaves_dict [ node ] , key = lambda n : n . wn_id ) # Build a list of relative class IDs of leaves that have at least # min_examples_in_class examples. ids_rel = [ ] for leaf in node_leaves : abs_id = dataset_spec . class_names_to_ids [ leaf . wn_id ] if abs_id in self . _filtered_class_set : ids_rel . append ( abs_to_rel_ids [ abs_id ] ) # Internal nodes are eligible if they span at least # `min_allowed_classes` and at most `max_eligible` leaves. if self . min_ways <= len ( ids_rel ) <= MAX_SPANNING_LEAVES_ELIGIBLE : self . span_leaves_rel . append ( ids_rel ) num_eligible_nodes = len ( self . span_leaves_rel ) if num_eligible_nodes < 1 : raise ValueError ( 'There are no classes eligible for participating in ' 'episodes. Consider changing the value of ' '`EpisodeDescriptionSampler.min_ways` in gin, or ' 'or MAX_SPANNING_LEAVES_ELIGIBLE in data.py.' )