(data_loader, func, name='')
return(np.mean(scores), dict(d0s=d0s,d1s=d1s,gts=gts,scores=scores))
Function computes Two Alternative Forced Choice (2AFC) score using
        distance function 'func' in dataset 'data_loader'
    INPUTS
        data_loader - CustomDatasetDataLoader object - contains a TwoAFCDataset inside
        func - callable distance function - calling d=func(in0,in1) should take 2
            pytorch tensors with shape Nx3xXxY, and return numpy array of length N
    OUTPUTS
        [0] - 2AFC score in [0,1], fraction of time func agrees with human evaluators
        [1] - dictionary with following elements
            d0s,d1s - N arrays containing distances between reference patch to perturbed patches 
            gts - N array in [0,1], preferred patch selected by human evaluators
                (closer to "0" for left patch p0, "1" for right patch p1,
                "0.6" means 60pct people preferred right patch, 40pct preferred left)
            scores - N array in [0,1], corresponding to what percentage function agreed with humans
    CONSTS
        N - number of test triplets in data_loader
Function computes Two Alternative Forced Choice (2AFC) score using
        distance function 'func' in dataset 'data_loader'
    INPUTS
        data_loader - CustomDatasetDataLoader object - contains a TwoAFCDataset inside
        func - callable distance function - calling d=func(in0,in1) should take 2
            pytorch tensors with shape Nx3xXxY, and return numpy array of length N
    OUTPUTS
        [0] - 2AFC score in [0,1], fraction of time func agrees with human evaluators
        [1] - dictionary with following elements
            d0s,d1s - N arrays containing distances between reference patch to perturbed patches 
            gts - N array in [0,1], preferred patch selected by human evaluators
                (closer to "0" for left patch p0, "1" for right patch p1,
                "0.6" means 60pct people preferred right patch, 40pct preferred left)
            scores - N array in [0,1], corresponding to what percentage function agreed with humans
    CONSTS
        N - number of test triplets in data_loader
Function computes Two Alternative Forced Choice ( 2AFC ) score using distance function func in dataset data_loader INPUTS data_loader - CustomDatasetDataLoader object - contains a TwoAFCDataset inside func - callable distance function - calling d = func ( in0 in1 ) should take 2 pytorch tensors with shape Nx3xXxY and return numpy array of length N OUTPUTS [ 0 ] - 2AFC score in [ 0 1 ] fraction of time func agrees with human evaluators [ 1 ] - dictionary with following elements d0s d1s - N arrays containing distances between reference patch to perturbed patches gts - N array in [ 0 1 ] preferred patch selected by human evaluators ( closer to 0 for left patch p0 1 for right patch p1 0 . 6 means 60pct people preferred right patch 40pct preferred left ) scores - N array in [ 0 1 ] corresponding to what percentage function agreed with humans CONSTS N - number of test triplets in data_loader
def score_2afc_dataset(data_loader, func, name=''):
    ''' Function computes Two Alternative Forced Choice (2AFC) score using
        distance function 'func' in dataset 'data_loader'
    INPUTS
        data_loader - CustomDatasetDataLoader object - contains a TwoAFCDataset inside
        func - callable distance function - calling d=func(in0,in1) should take 2
            pytorch tensors with shape Nx3xXxY, and return numpy array of length N
    OUTPUTS
        [0] - 2AFC score in [0,1], fraction of time func agrees with human evaluators
        [1] - dictionary with following elements
            d0s,d1s - N arrays containing distances between reference patch to perturbed patches 
            gts - N array in [0,1], preferred patch selected by human evaluators
                (closer to "0" for left patch p0, "1" for right patch p1,
                "0.6" means 60pct people preferred right patch, 40pct preferred left)
            scores - N array in [0,1], corresponding to what percentage function agreed with humans
    CONSTS
        N - number of test triplets in data_loader
    '''

    d0s = []
    d1s = []
    gts = []

    for data in tqdm(data_loader.load_data(), desc=name):
        d0s+=func(data['ref'],data['p0']).data.cpu().numpy().flatten().tolist()
        d1s+=func(data['ref'],data['p1']).data.cpu().numpy().flatten().tolist()
        gts+=data['judge'].cpu().numpy().flatten().tolist()

    d0s = np.array(d0s)
    d1s = np.array(d1s)
    gts = np.array(gts)
    scores = (d0s<d1s)*(1.-gts) + (d1s<d0s)*gts + (d1s==d0s)*.5

    return(np.mean(scores), dict(d0s=d0s,d1s=d1s,gts=gts,scores=scores))
def score_2afc_dataset ( data_loader , func , name = '' ) : d0s = [ ] d1s = [ ] gts = [ ] for data in tqdm ( data_loader . load_data ( ) , desc = name ) : d0s += func ( data [ 'ref' ] , data [ 'p0' ] ) . data . cpu ( ) . numpy ( ) . flatten ( ) . tolist ( ) d1s += func ( data [ 'ref' ] , data [ 'p1' ] ) . data . cpu ( ) . numpy ( ) . flatten ( ) . tolist ( ) gts += data [ 'judge' ] . cpu ( ) . numpy ( ) . flatten ( ) . tolist ( ) d0s = np . array ( d0s ) d1s = np . array ( d1s ) gts = np . array ( gts ) scores = ( d0s < d1s ) * ( 1. - gts ) + ( d1s < d0s ) * gts + ( d1s == d0s ) * .5 return ( np . mean ( scores ) , dict ( d0s = d0s , d1s = d1s , gts = gts , scores = scores ) )