(args)
Average predictions of different iterations and compute stats
Average predictions of different iterations and compute stats
Average predictions of different iterations and compute stats
def get_avg_stats(args):
    """Average predictions of different iterations and compute stats
    """

    test_hdf5_path = os.path.join(args.data_dir, "eval.h5")
    workspace = args.workspace
    filename = args.filename
    balance_type = args.balance_type
    model_type = args.model_type

    bgn_iteration = args.bgn_iteration
    fin_iteration = args.fin_iteration
    interval_iteration = args.interval_iteration

    get_avg_stats_time = time.time()

    # Load ground truth
    (test_x, test_y, test_id_list) = load_data(test_hdf5_path)
    target = test_y

    sub_dir = os.path.join(filename,
                           'balance_type={}'.format(balance_type),
                           'model_type={}'.format(model_type))

    # Average prediction probabilities of several iterations
    prob_dir = os.path.join(workspace, "probs", sub_dir, "test")
    prob_names = os.listdir(prob_dir)

    probs = []
    iterations = range(bgn_iteration, fin_iteration, interval_iteration)

    for iteration in iterations:

        pickle_path = os.path.join(prob_dir,
                                   "prob_{}_iters.p".format(iteration))

        prob = cPickle.load(open(pickle_path, 'rb'))
        probs.append(prob)

    avg_prob = np.mean(np.array(probs), axis=0)

    # Calculate stats
    stats = calculate_stats(avg_prob, target)

    logging.info("Callback time: {}".format(time.time() - get_avg_stats_time))

    # Write out to log
    mAP = np.mean([stat['AP'] for stat in stats])
    mAUC = np.mean([stat['auc'] for stat in stats])

    logging.info(
        "bgn_iteration, fin_iteration, interval_iteration: {}, {}, {}".format(
            bgn_iteration,
            fin_iteration,
            interval_iteration))

    logging.info("mAP: {:.6f}".format(mAP))
    logging.info("AUC: {:.6f}".format(mAUC))
    logging.info("d_prime: {:.6f}".format(d_prime(mAUC)))
def get_avg_stats ( args ) : test_hdf5_path = os . path . join ( args . data_dir , "eval.h5" ) workspace = args . workspace filename = args . filename balance_type = args . balance_type model_type = args . model_type bgn_iteration = args . bgn_iteration fin_iteration = args . fin_iteration interval_iteration = args . interval_iteration get_avg_stats_time = time . time ( ) # Load ground truth ( test_x , test_y , test_id_list ) = load_data ( test_hdf5_path ) target = test_y sub_dir = os . path . join ( filename , 'balance_type={}' . format ( balance_type ) , 'model_type={}' . format ( model_type ) ) # Average prediction probabilities of several iterations prob_dir = os . path . join ( workspace , "probs" , sub_dir , "test" ) prob_names = os . listdir ( prob_dir ) probs = [ ] iterations = range ( bgn_iteration , fin_iteration , interval_iteration ) for iteration in iterations : pickle_path = os . path . join ( prob_dir , "prob_{}_iters.p" . format ( iteration ) ) prob = cPickle . load ( open ( pickle_path , 'rb' ) ) probs . append ( prob ) avg_prob = np . mean ( np . array ( probs ) , axis = 0 ) # Calculate stats stats = calculate_stats ( avg_prob , target ) logging . info ( "Callback time: {}" . format ( time . time ( ) - get_avg_stats_time ) ) # Write out to log mAP = np . mean ( [ stat [ 'AP' ] for stat in stats ] ) mAUC = np . mean ( [ stat [ 'auc' ] for stat in stats ] ) logging . info ( "bgn_iteration, fin_iteration, interval_iteration: {}, {}, {}" . format ( bgn_iteration , fin_iteration , interval_iteration ) ) logging . info ( "mAP: {:.6f}" . format ( mAP ) ) logging . info ( "AUC: {:.6f}" . format ( mAUC ) ) logging . info ( "d_prime: {:.6f}" . format ( d_prime ( mAUC ) ) )