(split_enum,
                                  files_to_skip,
                                  path_to_num_leaf_images=None,
                                  train_split_only=False,
                                  log_stats=True)
return (splits, split_num_images, sampling_graph, synsets_2012,
          num_synset_2012_images, valid_test_roots)
Creates the dataset specification of ImageNet.

  This amounts to creating a data structure, a DAG specifically, whose nodes are
  synsets, and whose leaves are all and only the synsets corresponding to
  classes of the ILSVRC 2012 subset of ImageNet. Each synset is connected to
  others via child / parent links that indicate is-a relationships of the
  corresponding concepts (classes). This DAG will consist of all and only the
  nodes that are on paths starting from a leaf to a root (i.e. it won't contain
  any nodes that don't have leaf descendents) and it will be 'detached' from the
  remaining ImageNet synsets (that don't belong to the ILSVRC 2012 subset) in
  that by following a child or parent pointer of any DAG node we are guaranteed
  to reach another node that also belongs to the DAG.

  Args:
    split_enum: A class that inherits from enum.Enum whose attributes are TRAIN,
      VALID, and TEST, which are mapped to enumerated constants.
    files_to_skip: A set with the files that intersect with other datasets.
    path_to_num_leaf_images: A string, representing a path to a file containing
      a dict that maps the WordNet id of each ILSVRC 2012 class to the
      corresponding number of images. If no file is present, it will be created
      in order to save on future computation. If None, no attempt at reloading
      or storing the dict is made.
    train_split_only: bool, if True, we return the whole Imagenet as our
      training set.
    log_stats: whether to print statistics about the sampling graph and the
      three split subgraphs

  Returns:
    A tuple of the following:
    splits: A dict mapping each Split in split_enum to the list of Synsets
      belonging to the subgraph for that split.
    split_num_images: A dict mapping each Split in split_enum to a dict for the
      corresponding split that maps each node in its subgraph to the number of
      images in the subgraph of that node.
    sampling_graph: A set of the Synsets that belong to the DAG described above
    synsets_2012: The list of Synsets of classes of ILSVRC 2012
    num_synset_2012_images: A dict mapping each WordNet id of ILSVRC 2012 to its
      number of images
Creates the dataset specification of ImageNet.
Creates the dataset specification of ImageNet .
def create_imagenet_specification(split_enum,
                                  files_to_skip,
                                  path_to_num_leaf_images=None,
                                  train_split_only=False,
                                  log_stats=True):
  """Creates the dataset specification of ImageNet.

  This amounts to creating a data structure, a DAG specifically, whose nodes are
  synsets, and whose leaves are all and only the synsets corresponding to
  classes of the ILSVRC 2012 subset of ImageNet. Each synset is connected to
  others via child / parent links that indicate is-a relationships of the
  corresponding concepts (classes). This DAG will consist of all and only the
  nodes that are on paths starting from a leaf to a root (i.e. it won't contain
  any nodes that don't have leaf descendents) and it will be 'detached' from the
  remaining ImageNet synsets (that don't belong to the ILSVRC 2012 subset) in
  that by following a child or parent pointer of any DAG node we are guaranteed
  to reach another node that also belongs to the DAG.

  Args:
    split_enum: A class that inherits from enum.Enum whose attributes are TRAIN,
      VALID, and TEST, which are mapped to enumerated constants.
    files_to_skip: A set with the files that intersect with other datasets.
    path_to_num_leaf_images: A string, representing a path to a file containing
      a dict that maps the WordNet id of each ILSVRC 2012 class to the
      corresponding number of images. If no file is present, it will be created
      in order to save on future computation. If None, no attempt at reloading
      or storing the dict is made.
    train_split_only: bool, if True, we return the whole Imagenet as our
      training set.
    log_stats: whether to print statistics about the sampling graph and the
      three split subgraphs

  Returns:
    A tuple of the following:
    splits: A dict mapping each Split in split_enum to the list of Synsets
      belonging to the subgraph for that split.
    split_num_images: A dict mapping each Split in split_enum to a dict for the
      corresponding split that maps each node in its subgraph to the number of
      images in the subgraph of that node.
    sampling_graph: A set of the Synsets that belong to the DAG described above
    synsets_2012: The list of Synsets of classes of ILSVRC 2012
    num_synset_2012_images: A dict mapping each WordNet id of ILSVRC 2012 to its
      number of images
  """
  # Create Synsets for all ImageNet synsets (82115 in total).
  data_root = FLAGS.ilsvrc_2012_data_root
  synsets = {}
  path_to_words = FLAGS.path_to_words
  if not path_to_words:
    path_to_words = os.path.join(data_root, 'words.txt')
  with tf.io.gfile.GFile(path_to_words) as f:
    for line in f:
      wn_id, words = line.rstrip().split('\t')
      synsets[wn_id] = Synset(wn_id, words, set(), set())

  # Populate the parents / children arrays of these Synsets.
  path_to_is_a = FLAGS.path_to_is_a
  if not path_to_is_a:
    path_to_is_a = os.path.join(data_root, 'wordnet.is_a.txt')
  with tf.io.gfile.GFile(path_to_is_a, 'r') as f:
    for line in f:
      parent, child = line.rstrip().split(' ')
      synsets[parent].children.add(synsets[child])
      synsets[child].parents.add(synsets[parent])

  # Get the WordNet id's of the synsets of ILSVRC 2012.
  wn_ids_2012 = tf.io.gfile.listdir(data_root)
  wn_ids_2012 = set(
      entry for entry in wn_ids_2012
      if tf.io.gfile.isdir(os.path.join(data_root, entry)))
  synsets_2012 = [s for s in synsets.values() if s.wn_id in wn_ids_2012]
  assert len(wn_ids_2012) == len(synsets_2012)

  # Get a dict mapping each WordNet id of ILSVRC 2012 to its number of images.
  num_synset_2012_images = get_num_synset_2012_images(path_to_num_leaf_images,
                                                      synsets_2012,
                                                      files_to_skip)

  # Get the graph of all and only the ancestors of the ILSVRC 2012 classes.
  sampling_graph = create_sampling_graph(synsets_2012)

  # Create a dict mapping each node to its reachable leaves.
  spanning_leaves = get_spanning_leaves(sampling_graph)

  # Create a dict mapping each node in sampling graph to the number of images of
  # ILSVRC 2012 synsets that live in the sub-graph rooted at that node.
  num_images = get_num_spanning_images(spanning_leaves, num_synset_2012_images)

  if train_split_only:
    # We are keeping all graph for training.
    valid_test_roots = None
    splits = {
        split_enum.TRAIN: spanning_leaves,
        split_enum.VALID: set(),
        split_enum.TEST: set()
    }
  else:
    # Create class splits, each with its own sampling graph.
    # Choose roots for the validation and test subtrees (see the docstring of
    # create_splits for more information on how these are used).
    valid_test_roots = {
        'valid': get_synset_by_wnid('n02075296', sampling_graph),  # 'carnivore'
        'test':
            get_synset_by_wnid('n03183080', sampling_graph)  # 'device'
    }
    # The valid_test_roots returned here correspond to the same Synsets as in
    # the above dict, but are the copied versions of them for each subgraph.
    splits, valid_test_roots = create_splits(
        spanning_leaves, split_enum, valid_test_roots=valid_test_roots)

  # Compute num_images for each split.
  split_num_images = {}
  split_num_images[split_enum.TRAIN] = get_num_spanning_images(
      get_spanning_leaves(splits[split_enum.TRAIN]), num_synset_2012_images)
  split_num_images[split_enum.VALID] = get_num_spanning_images(
      get_spanning_leaves(splits[split_enum.VALID]), num_synset_2012_images)
  split_num_images[split_enum.TEST] = get_num_spanning_images(
      get_spanning_leaves(splits[split_enum.TEST]), num_synset_2012_images)

  # Compute statistics.
  if log_stats:
    imagenet_stats.log_graph_stats(
        sampling_graph,
        num_images,
        get_leaves,
        get_spanning_leaves,
        graph_name='all')
    imagenet_stats.log_graph_stats(
        splits[split_enum.TRAIN],
        split_num_images[split_enum.TRAIN],
        get_leaves,
        get_spanning_leaves,
        graph_name='train')
    imagenet_stats.log_graph_stats(
        splits[split_enum.VALID],
        split_num_images[split_enum.VALID],
        get_leaves,
        get_spanning_leaves,
        graph_name='valid')
    imagenet_stats.log_graph_stats(
        splits[split_enum.TEST],
        split_num_images[split_enum.TEST],
        get_leaves,
        get_spanning_leaves,
        graph_name='test')
    # Stats relevant to analysis of fine-graindness.
    imagenet_stats.log_stats_finegrainedness(
        splits[split_enum.TRAIN],
        get_leaves,
        get_lowest_common_ancestor,
        graph_name='train',
        path='longest')
    imagenet_stats.log_stats_finegrainedness(
        splits[split_enum.TEST],
        get_leaves,
        get_lowest_common_ancestor,
        graph_name='test',
        path='longest')

  # Note that spanning_leaves and num_images can easily be created from
  # sampling_graph if required.
  return (splits, split_num_images, sampling_graph, synsets_2012,
          num_synset_2012_images, valid_test_roots)
def create_imagenet_specification ( split_enum , files_to_skip , path_to_num_leaf_images = None , train_split_only = False , log_stats = True ) : # Create Synsets for all ImageNet synsets (82115 in total). data_root = FLAGS . ilsvrc_2012_data_root synsets = { } path_to_words = FLAGS . path_to_words if not path_to_words : path_to_words = os . path . join ( data_root , 'words.txt' ) with tf . io . gfile . GFile ( path_to_words ) as f : for line in f : wn_id , words = line . rstrip ( ) . split ( '\t' ) synsets [ wn_id ] = Synset ( wn_id , words , set ( ) , set ( ) ) # Populate the parents / children arrays of these Synsets. path_to_is_a = FLAGS . path_to_is_a if not path_to_is_a : path_to_is_a = os . path . join ( data_root , 'wordnet.is_a.txt' ) with tf . io . gfile . GFile ( path_to_is_a , 'r' ) as f : for line in f : parent , child = line . rstrip ( ) . split ( ' ' ) synsets [ parent ] . children . add ( synsets [ child ] ) synsets [ child ] . parents . add ( synsets [ parent ] ) # Get the WordNet id's of the synsets of ILSVRC 2012. wn_ids_2012 = tf . io . gfile . listdir ( data_root ) wn_ids_2012 = set ( entry for entry in wn_ids_2012 if tf . io . gfile . isdir ( os . path . join ( data_root , entry ) ) ) synsets_2012 = [ s for s in synsets . values ( ) if s . wn_id in wn_ids_2012 ] assert len ( wn_ids_2012 ) == len ( synsets_2012 ) # Get a dict mapping each WordNet id of ILSVRC 2012 to its number of images. num_synset_2012_images = get_num_synset_2012_images ( path_to_num_leaf_images , synsets_2012 , files_to_skip ) # Get the graph of all and only the ancestors of the ILSVRC 2012 classes. sampling_graph = create_sampling_graph ( synsets_2012 ) # Create a dict mapping each node to its reachable leaves. spanning_leaves = get_spanning_leaves ( sampling_graph ) # Create a dict mapping each node in sampling graph to the number of images of # ILSVRC 2012 synsets that live in the sub-graph rooted at that node. num_images = get_num_spanning_images ( spanning_leaves , num_synset_2012_images ) if train_split_only : # We are keeping all graph for training. valid_test_roots = None splits = { split_enum . TRAIN : spanning_leaves , split_enum . VALID : set ( ) , split_enum . TEST : set ( ) } else : # Create class splits, each with its own sampling graph. # Choose roots for the validation and test subtrees (see the docstring of # create_splits for more information on how these are used). valid_test_roots = { 'valid' : get_synset_by_wnid ( 'n02075296' , sampling_graph ) , # 'carnivore' 'test' : get_synset_by_wnid ( 'n03183080' , sampling_graph ) # 'device' } # The valid_test_roots returned here correspond to the same Synsets as in # the above dict, but are the copied versions of them for each subgraph. splits , valid_test_roots = create_splits ( spanning_leaves , split_enum , valid_test_roots = valid_test_roots ) # Compute num_images for each split. split_num_images = { } split_num_images [ split_enum . TRAIN ] = get_num_spanning_images ( get_spanning_leaves ( splits [ split_enum . TRAIN ] ) , num_synset_2012_images ) split_num_images [ split_enum . VALID ] = get_num_spanning_images ( get_spanning_leaves ( splits [ split_enum . VALID ] ) , num_synset_2012_images ) split_num_images [ split_enum . TEST ] = get_num_spanning_images ( get_spanning_leaves ( splits [ split_enum . TEST ] ) , num_synset_2012_images ) # Compute statistics. if log_stats : imagenet_stats . log_graph_stats ( sampling_graph , num_images , get_leaves , get_spanning_leaves , graph_name = 'all' ) imagenet_stats . log_graph_stats ( splits [ split_enum . TRAIN ] , split_num_images [ split_enum . TRAIN ] , get_leaves , get_spanning_leaves , graph_name = 'train' ) imagenet_stats . log_graph_stats ( splits [ split_enum . VALID ] , split_num_images [ split_enum . VALID ] , get_leaves , get_spanning_leaves , graph_name = 'valid' ) imagenet_stats . log_graph_stats ( splits [ split_enum . TEST ] , split_num_images [ split_enum . TEST ] , get_leaves , get_spanning_leaves , graph_name = 'test' ) # Stats relevant to analysis of fine-graindness. imagenet_stats . log_stats_finegrainedness ( splits [ split_enum . TRAIN ] , get_leaves , get_lowest_common_ancestor , graph_name = 'train' , path = 'longest' ) imagenet_stats . log_stats_finegrainedness ( splits [ split_enum . TEST ] , get_leaves , get_lowest_common_ancestor , graph_name = 'test' , path = 'longest' ) # Note that spanning_leaves and num_images can easily be created from # sampling_graph if required. return ( splits , split_num_images , sampling_graph , synsets_2012 , num_synset_2012_images , valid_test_roots )