(paths_to_event_files, experiment_root_dir,
                   do_finegrainedness_analysis, do_imbalance_analysis,
                   force_recompute)
Analyzes each event file and stores the .pklz in the corresponding dir.
Analyzes each event file and stores the .pklz in the corresponding dir.
Analyzes each event file and stores the . pklz in the corresponding dir .
def analyze_events(paths_to_event_files, experiment_root_dir,
                   do_finegrainedness_analysis, do_imbalance_analysis,
                   force_recompute):
  """Analyzes each event file and stores the .pklz in the corresponding dir."""

  # Aggregate stats across all event files and write (the summarized version of
  # those) to the root directory.
  shot_to_precision_all = []
  way_to_accuracy_all = []
  height_to_accuracy_all = []
  imbalance_to_accuracy_all = []

  for path_to_event in paths_to_event_files:
    output_pickle = get_output_path(path_to_event)

    # First check if the required data is already computed and written.
    maybe_data = False if force_recompute else read_pkl(output_pickle)
    if maybe_data:
      logging.info('Output %s already exists. Skipping it.', output_pickle)
      shot_to_precision = maybe_data['shot_to_precision']
      way_to_accuracy = maybe_data['way_to_accuracy']
      height_to_accuracy = maybe_data['height_to_accuracy']
      imbalance_to_accuracy = maybe_data['imbalance_to_accuracy']

    else:
      # Read the data from the event files.
      (ways, shots, class_props, class_ids, test_logits,
       test_targets) = read_data(path_to_event, do_finegrainedness_analysis,
                                 do_imbalance_analysis)

      # A dict mapping each observed 'shot' to a list of class precisions
      # obtained by classes that had that shot (regardless of shots of other
      # classes in the same episode).
      shot_to_precision = get_shot_to_precision(shots, test_logits,
                                                test_targets)

      # A dict mapping each observed 'way' of an episode to a list of accuracies
      # of the episodes with that way.
      way_to_accuracy = get_way_to_accuracy(ways, test_logits, test_targets)

      # A dict mapping the height of the lowest common ancestor of each pair of
      # leaves defining the binary classiifcation task to the task's accuracy.
      height_to_accuracy = {}
      if do_finegrainedness_analysis:
        height_to_accuracy = get_height_to_accuracy(class_ids, test_logits,
                                                    test_targets)

      # A dict mapping the degree of imabalance of tasks to their accuracy.
      imbalance_to_accuracy = {}
      if do_imbalance_analysis:
        imbalance_to_accuracy = get_imbalance_to_accuracy(
            class_props, test_logits, test_targets)

      # Keep only the mean and confidence intervals instead of the list of all
      # precisons or accuracies, resp. as the values of these dicts.
      shot_to_precision_summarized = summarize_values_stats(shot_to_precision)
      way_to_accuracy_summarized = summarize_values_stats(way_to_accuracy)
      height_to_accuracy_summarized = summarize_values_stats(height_to_accuracy)
      imbalance_to_accuracy_summarized = summarize_values_stats(
          imbalance_to_accuracy)

      # Save the two dicts to a pickle at the designated location.
      output_data = {
          'shot_to_precision': shot_to_precision,
          'way_to_accuracy': way_to_accuracy,
          'height_to_accuracy': height_to_accuracy,
          'imbalance_to_accuracy': imbalance_to_accuracy,
          'shot_to_precision_summarized': shot_to_precision_summarized,
          'way_to_accuracy_summarized': way_to_accuracy_summarized,
          'height_to_accuracy_summarized': height_to_accuracy_summarized,
          'imbalance_to_accuracy_summarized': imbalance_to_accuracy_summarized,
      }
      write_pkl(output_data, output_pickle)

    shot_to_precision_all.append(shot_to_precision)
    way_to_accuracy_all.append(way_to_accuracy)
    height_to_accuracy_all.append(height_to_accuracy)
    imbalance_to_accuracy_all.append(imbalance_to_accuracy)

  # Now aggregate the stats across datasets.
  shot_to_precision_all = combine_dicts(shot_to_precision_all)
  way_to_accuracy_all = combine_dicts(way_to_accuracy_all)
  height_to_accuracy_all = combine_dicts(height_to_accuracy_all)
  imbalance_to_accuracy_all = combine_dicts(imbalance_to_accuracy_all)

  # Summarize them.
  shot_to_precision_all_summarized = summarize_values_stats(
      shot_to_precision_all)
  way_to_accuracy_all_summarized = summarize_values_stats(way_to_accuracy_all)
  height_to_accuracy_all_summarized = summarize_values_stats(
      height_to_accuracy_all)
  imbalance_to_accuracy_all_summarized = summarize_values_stats(
      imbalance_to_accuracy_all)

  # Save the dicts to a pickle at the designated location.
  output_data = {
      'shot_to_precision': shot_to_precision_all,
      'way_to_accuracy': way_to_accuracy_all,
      'height_to_accuracy': height_to_accuracy_all,
      'shot_to_precision_summarized': shot_to_precision_all_summarized,
      'way_to_accuracy_summarized': way_to_accuracy_all_summarized,
      'height_to_accuracy_summarized': height_to_accuracy_all_summarized,
      'imbalance_to_accuracy_summarized': imbalance_to_accuracy_all_summarized
  }
  pickle_name_base = 'aggregated_summary_dicts'
  if int(FLAGS.restrict_to_subexperiment) > 0:
    pickle_name_base += '_eval_{}'.format(FLAGS.restrict_to_subexperiment)
  output_pickle = os.path.join(experiment_root_dir, pickle_name_base + '.pklz')
  write_pkl(output_data, output_pickle)
def analyze_events ( paths_to_event_files , experiment_root_dir , do_finegrainedness_analysis , do_imbalance_analysis , force_recompute ) : # Aggregate stats across all event files and write (the summarized version of # those) to the root directory. shot_to_precision_all = [ ] way_to_accuracy_all = [ ] height_to_accuracy_all = [ ] imbalance_to_accuracy_all = [ ] for path_to_event in paths_to_event_files : output_pickle = get_output_path ( path_to_event ) # First check if the required data is already computed and written. maybe_data = False if force_recompute else read_pkl ( output_pickle ) if maybe_data : logging . info ( 'Output %s already exists. Skipping it.' , output_pickle ) shot_to_precision = maybe_data [ 'shot_to_precision' ] way_to_accuracy = maybe_data [ 'way_to_accuracy' ] height_to_accuracy = maybe_data [ 'height_to_accuracy' ] imbalance_to_accuracy = maybe_data [ 'imbalance_to_accuracy' ] else : # Read the data from the event files. ( ways , shots , class_props , class_ids , test_logits , test_targets ) = read_data ( path_to_event , do_finegrainedness_analysis , do_imbalance_analysis ) # A dict mapping each observed 'shot' to a list of class precisions # obtained by classes that had that shot (regardless of shots of other # classes in the same episode). shot_to_precision = get_shot_to_precision ( shots , test_logits , test_targets ) # A dict mapping each observed 'way' of an episode to a list of accuracies # of the episodes with that way. way_to_accuracy = get_way_to_accuracy ( ways , test_logits , test_targets ) # A dict mapping the height of the lowest common ancestor of each pair of # leaves defining the binary classiifcation task to the task's accuracy. height_to_accuracy = { } if do_finegrainedness_analysis : height_to_accuracy = get_height_to_accuracy ( class_ids , test_logits , test_targets ) # A dict mapping the degree of imabalance of tasks to their accuracy. imbalance_to_accuracy = { } if do_imbalance_analysis : imbalance_to_accuracy = get_imbalance_to_accuracy ( class_props , test_logits , test_targets ) # Keep only the mean and confidence intervals instead of the list of all # precisons or accuracies, resp. as the values of these dicts. shot_to_precision_summarized = summarize_values_stats ( shot_to_precision ) way_to_accuracy_summarized = summarize_values_stats ( way_to_accuracy ) height_to_accuracy_summarized = summarize_values_stats ( height_to_accuracy ) imbalance_to_accuracy_summarized = summarize_values_stats ( imbalance_to_accuracy ) # Save the two dicts to a pickle at the designated location. output_data = { 'shot_to_precision' : shot_to_precision , 'way_to_accuracy' : way_to_accuracy , 'height_to_accuracy' : height_to_accuracy , 'imbalance_to_accuracy' : imbalance_to_accuracy , 'shot_to_precision_summarized' : shot_to_precision_summarized , 'way_to_accuracy_summarized' : way_to_accuracy_summarized , 'height_to_accuracy_summarized' : height_to_accuracy_summarized , 'imbalance_to_accuracy_summarized' : imbalance_to_accuracy_summarized , } write_pkl ( output_data , output_pickle ) shot_to_precision_all . append ( shot_to_precision ) way_to_accuracy_all . append ( way_to_accuracy ) height_to_accuracy_all . append ( height_to_accuracy ) imbalance_to_accuracy_all . append ( imbalance_to_accuracy ) # Now aggregate the stats across datasets. shot_to_precision_all = combine_dicts ( shot_to_precision_all ) way_to_accuracy_all = combine_dicts ( way_to_accuracy_all ) height_to_accuracy_all = combine_dicts ( height_to_accuracy_all ) imbalance_to_accuracy_all = combine_dicts ( imbalance_to_accuracy_all ) # Summarize them. shot_to_precision_all_summarized = summarize_values_stats ( shot_to_precision_all ) way_to_accuracy_all_summarized = summarize_values_stats ( way_to_accuracy_all ) height_to_accuracy_all_summarized = summarize_values_stats ( height_to_accuracy_all ) imbalance_to_accuracy_all_summarized = summarize_values_stats ( imbalance_to_accuracy_all ) # Save the dicts to a pickle at the designated location. output_data = { 'shot_to_precision' : shot_to_precision_all , 'way_to_accuracy' : way_to_accuracy_all , 'height_to_accuracy' : height_to_accuracy_all , 'shot_to_precision_summarized' : shot_to_precision_all_summarized , 'way_to_accuracy_summarized' : way_to_accuracy_all_summarized , 'height_to_accuracy_summarized' : height_to_accuracy_all_summarized , 'imbalance_to_accuracy_summarized' : imbalance_to_accuracy_all_summarized } pickle_name_base = 'aggregated_summary_dicts' if int ( FLAGS . restrict_to_subexperiment ) > 0 : pickle_name_base += '_eval_{}' . format ( FLAGS . restrict_to_subexperiment ) output_pickle = os . path . join ( experiment_root_dir , pickle_name_base + '.pklz' ) write_pkl ( output_data , output_pickle )