(stats_df, alg1, alg2)
return fig
Meta-analysis to compare two algorithms across several datasets

    A meta-analysis style plot that shows the standardized effect with
    confidence intervals over all datasets for two algorithms.
    Hypothesis is that alg1 is larger than alg2

    Parameters
    ----------
    stats_df: DataFrame
        DataFrame generated by compute_dataset_statistics
    alg1: str
        Name of first pipeline
    alg2: str
        Name of second pipeline

    Returns
    -------
    fig: Figure
        Pyplot handle
Meta-analysis to compare two algorithms across several datasets
Meta - analysis to compare two algorithms across several datasets
def meta_analysis_plot(stats_df, alg1, alg2):  # noqa: C901
    """Meta-analysis to compare two algorithms across several datasets

    A meta-analysis style plot that shows the standardized effect with
    confidence intervals over all datasets for two algorithms.
    Hypothesis is that alg1 is larger than alg2

    Parameters
    ----------
    stats_df: DataFrame
        DataFrame generated by compute_dataset_statistics
    alg1: str
        Name of first pipeline
    alg2: str
        Name of second pipeline

    Returns
    -------
    fig: Figure
        Pyplot handle
    """

    def _marker(pval):
        if pval < 0.001:
            return "$***$", 100
        elif pval < 0.01:
            return "$**$", 70
        elif pval < 0.05:
            return "$*$", 30
        else:
            raise ValueError("insignificant pval {}".format(pval))

    assert alg1 in stats_df.pipe1.unique()
    assert alg2 in stats_df.pipe1.unique()
    df_fw = stats_df.loc[(stats_df.pipe1 == alg1) & (stats_df.pipe2 == alg2)]
    df_fw = df_fw.sort_values(by="pipe1")
    df_bk = stats_df.loc[(stats_df.pipe1 == alg2) & (stats_df.pipe2 == alg1)]
    df_bk = df_bk.sort_values(by="pipe1")
    dsets = df_fw.dataset.unique()
    ci = []
    fig = plt.figure()
    gs = gridspec.GridSpec(1, 5)
    sig_ind = []
    pvals = []
    ax = fig.add_subplot(gs[0, :-1])
    ax.set_yticks(np.arange(len(dsets) + 1))
    ax.set_yticklabels(["Meta-effect"] + [_simplify_names(d) for d in dsets])
    pval_ax = fig.add_subplot(gs[0, -1], sharey=ax)
    plt.setp(pval_ax.get_yticklabels(), visible=False)
    _min = 0
    _max = 0
    for ind, d in enumerate(dsets):
        nsub = float(df_fw.loc[df_fw.dataset == d, "nsub"])
        t_dof = nsub - 1
        ci.append(t.ppf(0.95, t_dof) / np.sqrt(nsub))
        v = float(df_fw.loc[df_fw.dataset == d, "smd"])
        if v > 0:
            p = df_fw.loc[df_fw.dataset == d, "p"].item()
            if p < 0.05:
                sig_ind.append(ind)
                pvals.append(p)
        else:
            p = df_bk.loc[df_bk.dataset == d, "p"].item()
            if p < 0.05:
                sig_ind.append(ind)
                pvals.append(p)
        _min = _min if (_min < (v - ci[-1])) else (v - ci[-1])
        _max = _max if (_max > (v + ci[-1])) else (v + ci[-1])
        ax.plot(
            np.array([v - ci[-1], v + ci[-1]]), np.ones((2,)) * (ind + 1), c="tab:grey"
        )
    _range = max(abs(_min), abs(_max))
    ax.set_xlim((0 - _range, 0 + _range))
    final_effect = combine_effects(df_fw["smd"], df_fw["nsub"])
    ax.scatter(
        pd.concat([pd.Series([final_effect]), df_fw["smd"]]),
        np.arange(len(dsets) + 1),
        s=np.array([50] + [30] * len(dsets)),
        marker="D",
        c=["k"] + ["tab:grey"] * len(dsets),
    )
    for i, p in zip(sig_ind, pvals):
        m, s = _marker(p)
        ax.scatter(df_fw["smd"].iloc[i], i + 1.4, s=s, marker=m, color="r")
    # pvalues axis stuf
    pval_ax.set_xlim([-0.1, 0.1])
    pval_ax.grid(False)
    pval_ax.set_title("p-value", fontdict={"fontsize": 10})
    pval_ax.set_xticks([])
    for spine in pval_ax.spines.values():
        spine.set_visible(False)
    for ind, p in zip(sig_ind, pvals):
        pval_ax.text(
            0,
            ind + 1,
            horizontalalignment="center",
            verticalalignment="center",
            s="{:.2e}".format(p),
            fontsize=8,
        )
    if final_effect > 0:
        p = combine_pvalues(df_fw["p"], df_fw["nsub"])
        if p < 0.05:
            m, s = _marker(p)
            ax.scatter([final_effect], [-0.4], s=s, marker=m, c="r")
            pval_ax.text(
                0,
                0,
                horizontalalignment="center",
                verticalalignment="center",
                s="{:.2e}".format(p),
                fontsize=8,
            )
    else:
        p = combine_pvalues(df_bk["p"], df_bk["nsub"])
        if p < 0.05:
            m, s = _marker(p)
            ax.scatter([final_effect], [-0.4], s=s, marker=m, c="r")
            pval_ax.text(
                0,
                0,
                horizontalalignment="center",
                verticalalignment="center",
                s="{:.2e}".format(p),
                fontsize=8,
            )

    ax.grid(False)
    ax.spines["top"].set_visible(False)
    ax.spines["right"].set_visible(False)
    ax.axvline(0, linestyle="--", c="k")
    ax.axhline(0.5, linestyle="-", linewidth=3, c="k")
    title = "< {} better{}\n{}{} better >".format(
        alg2, " " * (45 - len(alg2)), " " * (45 - len(alg1)), alg1
    )
    ax.set_title(title, ha="left", ma="right", loc="left")
    ax.set_xlabel("Standardized Mean Difference")
    fig.tight_layout()

    return fig
def meta_analysis_plot ( stats_df , alg1 , alg2 ) : # noqa: C901 def _marker ( pval ) : if pval < 0.001 : return "$***$" , 100 elif pval < 0.01 : return "$**$" , 70 elif pval < 0.05 : return "$*$" , 30 else : raise ValueError ( "insignificant pval {}" . format ( pval ) ) assert alg1 in stats_df . pipe1 . unique ( ) assert alg2 in stats_df . pipe1 . unique ( ) df_fw = stats_df . loc [ ( stats_df . pipe1 == alg1 ) & ( stats_df . pipe2 == alg2 ) ] df_fw = df_fw . sort_values ( by = "pipe1" ) df_bk = stats_df . loc [ ( stats_df . pipe1 == alg2 ) & ( stats_df . pipe2 == alg1 ) ] df_bk = df_bk . sort_values ( by = "pipe1" ) dsets = df_fw . dataset . unique ( ) ci = [ ] fig = plt . figure ( ) gs = gridspec . GridSpec ( 1 , 5 ) sig_ind = [ ] pvals = [ ] ax = fig . add_subplot ( gs [ 0 , : - 1 ] ) ax . set_yticks ( np . arange ( len ( dsets ) + 1 ) ) ax . set_yticklabels ( [ "Meta-effect" ] + [ _simplify_names ( d ) for d in dsets ] ) pval_ax = fig . add_subplot ( gs [ 0 , - 1 ] , sharey = ax ) plt . setp ( pval_ax . get_yticklabels ( ) , visible = False ) _min = 0 _max = 0 for ind , d in enumerate ( dsets ) : nsub = float ( df_fw . loc [ df_fw . dataset == d , "nsub" ] ) t_dof = nsub - 1 ci . append ( t . ppf ( 0.95 , t_dof ) / np . sqrt ( nsub ) ) v = float ( df_fw . loc [ df_fw . dataset == d , "smd" ] ) if v > 0 : p = df_fw . loc [ df_fw . dataset == d , "p" ] . item ( ) if p < 0.05 : sig_ind . append ( ind ) pvals . append ( p ) else : p = df_bk . loc [ df_bk . dataset == d , "p" ] . item ( ) if p < 0.05 : sig_ind . append ( ind ) pvals . append ( p ) _min = _min if ( _min < ( v - ci [ - 1 ] ) ) else ( v - ci [ - 1 ] ) _max = _max if ( _max > ( v + ci [ - 1 ] ) ) else ( v + ci [ - 1 ] ) ax . plot ( np . array ( [ v - ci [ - 1 ] , v + ci [ - 1 ] ] ) , np . ones ( ( 2 , ) ) * ( ind + 1 ) , c = "tab:grey" ) _range = max ( abs ( _min ) , abs ( _max ) ) ax . set_xlim ( ( 0 - _range , 0 + _range ) ) final_effect = combine_effects ( df_fw [ "smd" ] , df_fw [ "nsub" ] ) ax . scatter ( pd . concat ( [ pd . Series ( [ final_effect ] ) , df_fw [ "smd" ] ] ) , np . arange ( len ( dsets ) + 1 ) , s = np . array ( [ 50 ] + [ 30 ] * len ( dsets ) ) , marker = "D" , c = [ "k" ] + [ "tab:grey" ] * len ( dsets ) , ) for i , p in zip ( sig_ind , pvals ) : m , s = _marker ( p ) ax . scatter ( df_fw [ "smd" ] . iloc [ i ] , i + 1.4 , s = s , marker = m , color = "r" ) # pvalues axis stuf pval_ax . set_xlim ( [ - 0.1 , 0.1 ] ) pval_ax . grid ( False ) pval_ax . set_title ( "p-value" , fontdict = { "fontsize" : 10 } ) pval_ax . set_xticks ( [ ] ) for spine in pval_ax . spines . values ( ) : spine . set_visible ( False ) for ind , p in zip ( sig_ind , pvals ) : pval_ax . text ( 0 , ind + 1 , horizontalalignment = "center" , verticalalignment = "center" , s = "{:.2e}" . format ( p ) , fontsize = 8 , ) if final_effect > 0 : p = combine_pvalues ( df_fw [ "p" ] , df_fw [ "nsub" ] ) if p < 0.05 : m , s = _marker ( p ) ax . scatter ( [ final_effect ] , [ - 0.4 ] , s = s , marker = m , c = "r" ) pval_ax . text ( 0 , 0 , horizontalalignment = "center" , verticalalignment = "center" , s = "{:.2e}" . format ( p ) , fontsize = 8 , ) else : p = combine_pvalues ( df_bk [ "p" ] , df_bk [ "nsub" ] ) if p < 0.05 : m , s = _marker ( p ) ax . scatter ( [ final_effect ] , [ - 0.4 ] , s = s , marker = m , c = "r" ) pval_ax . text ( 0 , 0 , horizontalalignment = "center" , verticalalignment = "center" , s = "{:.2e}" . format ( p ) , fontsize = 8 , ) ax . grid ( False ) ax . spines [ "top" ] . set_visible ( False ) ax . spines [ "right" ] . set_visible ( False ) ax . axvline ( 0 , linestyle = "--" , c = "k" ) ax . axhline ( 0.5 , linestyle = "-" , linewidth = 3 , c = "k" ) title = "< {} better{}\n{}{} better >" . format ( alg2 , " " * ( 45 - len ( alg2 ) ) , " " * ( 45 - len ( alg1 ) ) , alg1 ) ax . set_title ( title , ha = "left" , ma = "right" , loc = "left" ) ax . set_xlabel ( "Standardized Mean Difference" ) fig . tight_layout ( ) return fig