(df, perm_cutoff=20)
return dfP, dfT
Compute differences between pipelines across datasets

    Compute matrix of p-values for all algorithms over all datasets via
    combined p-values method

    Parameters
    ----------
    df: DataFrame
        Table of effect and p-values for each dataset and all pipelines, returned by
        compute_dataset_statistics
    perm_cutoff: int, default=20
        threshold value  to stop using permutation tests, which can be very expensive
        computationally, using Wilcoxon rank-sum test instead

    Returns
    -------
    dfP: DataFrame of shape (n_pipelines, n_pipelines)
        p-values per algorithm pairs
    dfT: DataFrame of shape (n_pipelines, n_pipelines)
        signed standardized mean differences
Compute differences between pipelines across datasets
Compute differences between pipelines across datasets
def find_significant_differences(df, perm_cutoff=20):
    """Compute differences between pipelines across datasets

    Compute matrix of p-values for all algorithms over all datasets via
    combined p-values method

    Parameters
    ----------
    df: DataFrame
        Table of effect and p-values for each dataset and all pipelines, returned by
        compute_dataset_statistics
    perm_cutoff: int, default=20
        threshold value  to stop using permutation tests, which can be very expensive
        computationally, using Wilcoxon rank-sum test instead

    Returns
    -------
    dfP: DataFrame of shape (n_pipelines, n_pipelines)
        p-values per algorithm pairs
    dfT: DataFrame of shape (n_pipelines, n_pipelines)
        signed standardized mean differences
    """
    dsets = df.dataset.unique()
    algs = df.pipe1.unique()
    nsubs = np.array([df.loc[df.dataset == d, "nsub"].mean() for d in dsets])
    P_full = df.pivot_table(values="p", index=["dataset", "pipe1"], columns="pipe2")
    T_full = df.pivot_table(values="smd", index=["dataset", "pipe1"], columns="pipe2")
    P = np.full((len(algs), len(algs)), np.NaN)
    T = np.full((len(algs), len(algs)), np.NaN)
    for i in range(len(algs)):
        for j in range(len(algs)):
            if i != j:
                p = P_full.loc[(slice(None), algs[i]), algs[j]]
                t = T_full.loc[(slice(None), algs[i]), algs[j]]
                P[i, j] = combine_pvalues(p, nsubs)
                if np.isnan(P[i, j]):
                    log.info("NaN p-value found, turned to 1")
                    print("NaN")
                    # P[i, j] = 1.0
                T[i, j] = combine_effects(t, nsubs)
    dfP = pd.DataFrame(index=algs, columns=algs, data=P)
    dfT = pd.DataFrame(index=algs, columns=algs, data=T)
    return dfP, dfT
def find_significant_differences ( df , perm_cutoff = 20 ) : dsets = df . dataset . unique ( ) algs = df . pipe1 . unique ( ) nsubs = np . array ( [ df . loc [ df . dataset == d , "nsub" ] . mean ( ) for d in dsets ] ) P_full = df . pivot_table ( values = "p" , index = [ "dataset" , "pipe1" ] , columns = "pipe2" ) T_full = df . pivot_table ( values = "smd" , index = [ "dataset" , "pipe1" ] , columns = "pipe2" ) P = np . full ( ( len ( algs ) , len ( algs ) ) , np . NaN ) T = np . full ( ( len ( algs ) , len ( algs ) ) , np . NaN ) for i in range ( len ( algs ) ) : for j in range ( len ( algs ) ) : if i != j : p = P_full . loc [ ( slice ( None ) , algs [ i ] ) , algs [ j ] ] t = T_full . loc [ ( slice ( None ) , algs [ i ] ) , algs [ j ] ] P [ i , j ] = combine_pvalues ( p , nsubs ) if np . isnan ( P [ i , j ] ) : log . info ( "NaN p-value found, turned to 1" ) print ( "NaN" ) # P[i, j] = 1.0 T [ i , j ] = combine_effects ( t , nsubs ) dfP = pd . DataFrame ( index = algs , columns = algs , data = P ) dfT = pd . DataFrame ( index = algs , columns = algs , data = T ) return dfP , dfT