| import os.path as osp | |
| from argparse import ArgumentParser | |
| import mmcv | |
| import numpy as np | |
| def print_coco_results(results): | |
| def _print(result, ap=1, iouThr=None, areaRng='all', maxDets=100): | |
| titleStr = 'Average Precision' if ap == 1 else 'Average Recall' | |
| typeStr = '(AP)' if ap == 1 else '(AR)' | |
| iouStr = '0.50:0.95' \ | |
| if iouThr is None else f'{iouThr:0.2f}' | |
| iStr = f' {titleStr:<18} {typeStr} @[ IoU={iouStr:<9} | ' | |
| iStr += f'area={areaRng:>6s} | maxDets={maxDets:>3d} ] = {result:0.3f}' | |
| print(iStr) | |
| stats = np.zeros((12, )) | |
| stats[0] = _print(results[0], 1) | |
| stats[1] = _print(results[1], 1, iouThr=.5) | |
| stats[2] = _print(results[2], 1, iouThr=.75) | |
| stats[3] = _print(results[3], 1, areaRng='small') | |
| stats[4] = _print(results[4], 1, areaRng='medium') | |
| stats[5] = _print(results[5], 1, areaRng='large') | |
| stats[6] = _print(results[6], 0, maxDets=1) | |
| stats[7] = _print(results[7], 0, maxDets=10) | |
| stats[8] = _print(results[8], 0) | |
| stats[9] = _print(results[9], 0, areaRng='small') | |
| stats[10] = _print(results[10], 0, areaRng='medium') | |
| stats[11] = _print(results[11], 0, areaRng='large') | |
| def get_coco_style_results(filename, | |
| task='bbox', | |
| metric=None, | |
| prints='mPC', | |
| aggregate='benchmark'): | |
| assert aggregate in ['benchmark', 'all'] | |
| if prints == 'all': | |
| prints = ['P', 'mPC', 'rPC'] | |
| elif isinstance(prints, str): | |
| prints = [prints] | |
| for p in prints: | |
| assert p in ['P', 'mPC', 'rPC'] | |
| if metric is None: | |
| metrics = [ | |
| 'AP', 'AP50', 'AP75', 'APs', 'APm', 'APl', 'AR1', 'AR10', 'AR100', | |
| 'ARs', 'ARm', 'ARl' | |
| ] | |
| elif isinstance(metric, list): | |
| metrics = metric | |
| else: | |
| metrics = [metric] | |
| for metric_name in metrics: | |
| assert metric_name in [ | |
| 'AP', 'AP50', 'AP75', 'APs', 'APm', 'APl', 'AR1', 'AR10', 'AR100', | |
| 'ARs', 'ARm', 'ARl' | |
| ] | |
| eval_output = mmcv.load(filename) | |
| num_distortions = len(list(eval_output.keys())) | |
| results = np.zeros((num_distortions, 6, len(metrics)), dtype='float32') | |
| for corr_i, distortion in enumerate(eval_output): | |
| for severity in eval_output[distortion]: | |
| for metric_j, metric_name in enumerate(metrics): | |
| mAP = eval_output[distortion][severity][task][metric_name] | |
| results[corr_i, severity, metric_j] = mAP | |
| P = results[0, 0, :] | |
| if aggregate == 'benchmark': | |
| mPC = np.mean(results[:15, 1:, :], axis=(0, 1)) | |
| else: | |
| mPC = np.mean(results[:, 1:, :], axis=(0, 1)) | |
| rPC = mPC / P | |
| print(f'\nmodel: {osp.basename(filename)}') | |
| if metric is None: | |
| if 'P' in prints: | |
| print(f'Performance on Clean Data [P] ({task})') | |
| print_coco_results(P) | |
| if 'mPC' in prints: | |
| print(f'Mean Performance under Corruption [mPC] ({task})') | |
| print_coco_results(mPC) | |
| if 'rPC' in prints: | |
| print(f'Relative Performance under Corruption [rPC] ({task})') | |
| print_coco_results(rPC) | |
| else: | |
| if 'P' in prints: | |
| print(f'Performance on Clean Data [P] ({task})') | |
| for metric_i, metric_name in enumerate(metrics): | |
| print(f'{metric_name:5} = {P[metric_i]:0.3f}') | |
| if 'mPC' in prints: | |
| print(f'Mean Performance under Corruption [mPC] ({task})') | |
| for metric_i, metric_name in enumerate(metrics): | |
| print(f'{metric_name:5} = {mPC[metric_i]:0.3f}') | |
| if 'rPC' in prints: | |
| print(f'Relative Performance under Corruption [rPC] ({task})') | |
| for metric_i, metric_name in enumerate(metrics): | |
| print(f'{metric_name:5} => {rPC[metric_i] * 100:0.1f} %') | |
| return results | |
| def get_voc_style_results(filename, prints='mPC', aggregate='benchmark'): | |
| assert aggregate in ['benchmark', 'all'] | |
| if prints == 'all': | |
| prints = ['P', 'mPC', 'rPC'] | |
| elif isinstance(prints, str): | |
| prints = [prints] | |
| for p in prints: | |
| assert p in ['P', 'mPC', 'rPC'] | |
| eval_output = mmcv.load(filename) | |
| num_distortions = len(list(eval_output.keys())) | |
| results = np.zeros((num_distortions, 6, 20), dtype='float32') | |
| for i, distortion in enumerate(eval_output): | |
| for severity in eval_output[distortion]: | |
| mAP = [ | |
| eval_output[distortion][severity][j]['ap'] | |
| for j in range(len(eval_output[distortion][severity])) | |
| ] | |
| results[i, severity, :] = mAP | |
| P = results[0, 0, :] | |
| if aggregate == 'benchmark': | |
| mPC = np.mean(results[:15, 1:, :], axis=(0, 1)) | |
| else: | |
| mPC = np.mean(results[:, 1:, :], axis=(0, 1)) | |
| rPC = mPC / P | |
| print(f'\nmodel: {osp.basename(filename)}') | |
| if 'P' in prints: | |
| print(f'Performance on Clean Data [P] in AP50 = {np.mean(P):0.3f}') | |
| if 'mPC' in prints: | |
| print('Mean Performance under Corruption [mPC] in AP50 = ' | |
| f'{np.mean(mPC):0.3f}') | |
| if 'rPC' in prints: | |
| print('Relative Performance under Corruption [rPC] in % = ' | |
| f'{np.mean(rPC) * 100:0.1f}') | |
| return np.mean(results, axis=2, keepdims=True) | |
| def get_results(filename, | |
| dataset='coco', | |
| task='bbox', | |
| metric=None, | |
| prints='mPC', | |
| aggregate='benchmark'): | |
| assert dataset in ['coco', 'voc', 'cityscapes'] | |
| if dataset in ['coco', 'cityscapes']: | |
| results = get_coco_style_results( | |
| filename, | |
| task=task, | |
| metric=metric, | |
| prints=prints, | |
| aggregate=aggregate) | |
| elif dataset == 'voc': | |
| if task != 'bbox': | |
| print('Only bbox analysis is supported for Pascal VOC') | |
| print('Will report bbox results\n') | |
| if metric not in [None, ['AP'], ['AP50']]: | |
| print('Only the AP50 metric is supported for Pascal VOC') | |
| print('Will report AP50 metric\n') | |
| results = get_voc_style_results( | |
| filename, prints=prints, aggregate=aggregate) | |
| return results | |
| def get_distortions_from_file(filename): | |
| eval_output = mmcv.load(filename) | |
| return get_distortions_from_results(eval_output) | |
| def get_distortions_from_results(eval_output): | |
| distortions = [] | |
| for i, distortion in enumerate(eval_output): | |
| distortions.append(distortion.replace('_', ' ')) | |
| return distortions | |
| def main(): | |
| parser = ArgumentParser(description='Corruption Result Analysis') | |
| parser.add_argument('filename', help='result file path') | |
| parser.add_argument( | |
| '--dataset', | |
| type=str, | |
| choices=['coco', 'voc', 'cityscapes'], | |
| default='coco', | |
| help='dataset type') | |
| parser.add_argument( | |
| '--task', | |
| type=str, | |
| nargs='+', | |
| choices=['bbox', 'segm'], | |
| default=['bbox'], | |
| help='task to report') | |
| parser.add_argument( | |
| '--metric', | |
| nargs='+', | |
| choices=[ | |
| None, 'AP', 'AP50', 'AP75', 'APs', 'APm', 'APl', 'AR1', 'AR10', | |
| 'AR100', 'ARs', 'ARm', 'ARl' | |
| ], | |
| default=None, | |
| help='metric to report') | |
| parser.add_argument( | |
| '--prints', | |
| type=str, | |
| nargs='+', | |
| choices=['P', 'mPC', 'rPC'], | |
| default='mPC', | |
| help='corruption benchmark metric to print') | |
| parser.add_argument( | |
| '--aggregate', | |
| type=str, | |
| choices=['all', 'benchmark'], | |
| default='benchmark', | |
| help='aggregate all results or only those \ | |
| for benchmark corruptions') | |
| args = parser.parse_args() | |
| for task in args.task: | |
| get_results( | |
| args.filename, | |
| dataset=args.dataset, | |
| task=task, | |
| metric=args.metric, | |
| prints=args.prints, | |
| aggregate=args.aggregate) | |
| if __name__ == '__main__': | |
| main() | |