<!-- Generated by pkgdown: do not edit by hand -->
<!DOCTYPE html>
<html lang="en">
  <head>
  <meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1.0">

<title>Fairness check — fairness_check • fairmodels</title>

<!-- favicons -->
<link rel="icon" type="image/png" sizes="16x16" href="../favicon-16x16.png">
<link rel="icon" type="image/png" sizes="32x32" href="../favicon-32x32.png">
<link rel="apple-touch-icon" type="image/png" sizes="180x180" href="../apple-touch-icon.png" />
<link rel="apple-touch-icon" type="image/png" sizes="120x120" href="../apple-touch-icon-120x120.png" />
<link rel="apple-touch-icon" type="image/png" sizes="76x76" href="../apple-touch-icon-76x76.png" />
<link rel="apple-touch-icon" type="image/png" sizes="60x60" href="../apple-touch-icon-60x60.png" />

<!-- jquery -->
<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
<!-- Bootstrap -->

<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/css/bootstrap.min.css" integrity="sha256-bZLfwXAP04zRMK2BjiO8iu9pf4FbLqX6zitd+tIvLhE=" crossorigin="anonymous" />

<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>

<!-- bootstrap-toc -->
<link rel="stylesheet" href="../bootstrap-toc.css">
<script src="../bootstrap-toc.js"></script>

<!-- Font Awesome icons -->
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/all.min.css" integrity="sha256-mmgLkCYLUQbXn0B1SRqzHar6dCnv9oZFPEC1g1cwlkk=" crossorigin="anonymous" />
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.12.1/css/v4-shims.min.css" integrity="sha256-wZjR52fzng1pJHwx4aV2AO3yyTOXrcDW7jBpJtTwVxw=" crossorigin="anonymous" />

<!-- clipboard.js -->
<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>

<!-- headroom.js -->
<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/headroom.min.js" integrity="sha256-AsUX4SJE1+yuDu5+mAVzJbuYNPHj/WroHuZ8Ir/CkE0=" crossorigin="anonymous"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/headroom/0.11.0/jQuery.headroom.min.js" integrity="sha256-ZX/yNShbjqsohH1k95liqY9Gd8uOiE1S4vZc+9KQ1K4=" crossorigin="anonymous"></script>

<!-- dalexverse -->
<link href="../dalexverse.css" rel="stylesheet">
<link href="../dalexverse-2.css" rel="stylesheet">

<!-- pkgdown -->
<link href="../pkgdown.css" rel="stylesheet">
<script src="../pkgdown.js"></script>






<meta property="og:title" content="Fairness check — fairness_check" />
<meta property="og:description" content="Fairness check creates fairness_object which measures different fairness metrics and wraps data, explainers and parameters in useful object. This is fundamental object in this package.
It enables to visualize fairness metrics and models in many ways and compare models on both fairness and performance level. Fairness check acts as merger and wrapper for explainers and fairness objects.
While other fairness objects values are not changed, fairness check assigns cutoffs and labels to provided explainers so same explainers with changed labels/cutoffs might be gradually added to fairness object.
Users through print and plot methods may quickly check values of most popular fairness metrics. More on that topic in details." />
<meta property="og:image" content="/logo.png" />

<meta name="robots" content="noindex" />


<!-- mathjax -->
<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js" integrity="sha256-nvJJv9wWKEm88qvoQl9ekL2J+k/RWIsaSScxxlsrv8k=" crossorigin="anonymous"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/config/TeX-AMS-MML_HTMLorMML.js" integrity="sha256-84DKXVJXs0/F8OTMzX4UR909+jtl4G7SPypPavF+GfA=" crossorigin="anonymous"></script>

<!--[if lt IE 9]>
<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
<![endif]-->

<!-- google analytics -->
<script async src="https://www.googletagmanager.com/gtag/js?id=UA-5650686-14""></script>
<script>
 window.dataLayer = window.dataLayer || [];
 function gtag(){dataLayer.push(arguments);}
 gtag('js', new Date());

 gtag('config', 'UA-5650686-14');
</script>

  </head>

  <body data-spy="scroll" data-target="#toc">
    <div class="container template-reference-topic">
      <header>
      <div class="navbar navbar-default navbar-fixed-top" role="navigation">
  <div class="container">
    <div class="navbar-header">
      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false">
        <span class="sr-only">Toggle navigation</span>
        <span class="icon-bar"></span>
        <span class="icon-bar"></span>
        <span class="icon-bar"></span>
      </button>

      <div class="navbar-brand-container">
        <a class="navbar-brand" href="../index.html">fairmodels</a>
        <div class="info">
          <span class="partof">part of the <a href="https://github.com/ModelOriented/DrWhy">DrWhy.AI</a>
           developed by the <a href="https://mi2.mini.pw.edu.pl/">MI^2 DataLab</a> </span>
          <span class="version version-default">0.1.1</span>
        </div>
      </div>
    </div>
    <div id="navbar" class="navbar-collapse collapse">
      <ul class="nav navbar-nav navbar-right">
        <li>
  <a href="../index.html">
    <span class="fas fa fas fa-home fa-lg"></span>
     
  </a>
</li>
<li>
  <a href="../reference/index.html">Reference</a>
</li>
<li class="dropdown">
  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
    Articles
     
    <span class="caret"></span>
  </a>
  <ul class="dropdown-menu" role="menu">
    <li>
      <a href="../articles/Advanced_tutorial.html">Advanced Tutorial</a>
    </li>
    <li>
      <a href="../articles/Basic_tutorial.html">Basic Tutorial</a>
    </li>
  </ul>
</li>
<li>
  <a href="../news/index.html">Changelog</a>
</li>
        <li>
  <a href="https://github.com/ModelOriented/fairmodels/">
    <span class="fab fa fab fa-github fa-lg"></span>
     
  </a>
</li>
      </ul>
      
    </div><!--/.nav-collapse -->
  </div><!--/.container -->
</div><!--/.navbar -->

      

      </header>

<div class="row">
  <div class="col-md-9 contents">
    <div class="page-header">
    <h1>Fairness check</h1>
    <small class="dont-index">Source: <a href='https://github.com/ModelOriented/fairmodels/blob/master/R/fairness_check.R'><code>R/fairness_check.R</code></a></small>
    <div class="hidden name"><code>fairness_check.Rd</code></div>
    </div>

    <div class="ref-description">
    <p>Fairness check creates <code>fairness_object</code> which measures different fairness metrics and wraps data, explainers and parameters in useful object. This is fundamental object in this package.
It enables to visualize fairness metrics and models in many ways and compare models on both fairness and performance level. Fairness check acts as merger and wrapper for explainers and fairness objects.
While other fairness objects values are not changed, fairness check assigns cutoffs and labels to provided explainers so same explainers with changed labels/cutoffs might be gradually added to fairness object.
Users through print and plot methods may quickly check values of most popular fairness metrics. More on that topic in details.</p>
    </div>

    <pre class="usage"><span class='fu'>fairness_check</span>(
  <span class='no'>x</span>,
  <span class='no'>...</span>,
  <span class='kw'>protected</span> <span class='kw'>=</span> <span class='kw'>NULL</span>,
  <span class='kw'>privileged</span> <span class='kw'>=</span> <span class='kw'>NULL</span>,
  <span class='kw'>cutoff</span> <span class='kw'>=</span> <span class='kw'>NULL</span>,
  <span class='kw'>label</span> <span class='kw'>=</span> <span class='kw'>NULL</span>,
  <span class='kw'>epsilon</span> <span class='kw'>=</span> <span class='kw'>NULL</span>,
  <span class='kw'>verbose</span> <span class='kw'>=</span> <span class='fl'>TRUE</span>,
  <span class='kw'>colorize</span> <span class='kw'>=</span> <span class='fl'>TRUE</span>
)</pre>

    <h2 class="hasAnchor" id="arguments"><a class="anchor" href="#arguments"></a>Arguments</h2>
    <table class="ref-arguments">
    <colgroup><col class="name" /><col class="desc" /></colgroup>
    <tr>
      <th>x</th>
      <td><p>object created with <code><a href='https://rdrr.io/pkg/DALEX/man/explain.html'>explain</a></code> or of class <code>fairness_object</code></p></td>
    </tr>
    <tr>
      <th>...</th>
      <td><p>possibly more objects created with <code><a href='https://rdrr.io/pkg/DALEX/man/explain.html'>explain</a></code> and/or objects of class <code>fairness_object</code></p></td>
    </tr>
    <tr>
      <th>protected</th>
      <td><p>factor, protected variable (also called sensitive attribute), containing privileged and unprivileged groups</p></td>
    </tr>
    <tr>
      <th>privileged</th>
      <td><p>factor/character, one value of <code>protected</code>, in regard to what subgroup parity loss is calculated</p></td>
    </tr>
    <tr>
      <th>cutoff</th>
      <td><p>numeric, vector of cutoffs (thresholds) for each value of protected variable, affecting only explainers.</p></td>
    </tr>
    <tr>
      <th>label</th>
      <td><p>character, vector of labels to be assigned for explainers, default is explainer label.</p></td>
    </tr>
    <tr>
      <th>epsilon</th>
      <td><p>numeric, boundary for fairness checking</p></td>
    </tr>
    <tr>
      <th>verbose</th>
      <td><p>logical, whether to print information about creation of fairness object</p></td>
    </tr>
    <tr>
      <th>colorize</th>
      <td><p>logical, whether to print information in color</p></td>
    </tr>
    </table>

    <h2 class="hasAnchor" id="value"><a class="anchor" href="#value"></a>Value</h2>

    <p>An object of class <code>fairness_object</code> which is a list with elements:</p><ul>
<li><p>parity_loss_metric_data - data.frame containing parity loss for various fairness metrics. Created with following metrics:</p><ul>
<li><p>TPR - True Positive Rate (Sensitivity, Recall)</p></li>
<li><p>TNR - True Negative Rate (Specificity)</p></li>
<li><p>PPV - Positive Predictive Value (Precision)</p></li>
<li><p>NPV - Negative Predictive Value</p></li>
<li><p>FNR - False Negative Rate</p></li>
<li><p>FPR - False Positive Rate</p></li>
<li><p>FDR - False Discovery Rate</p></li>
<li><p>FOR - False Omission Rate</p></li>
<li><p>TS - Threat Score</p></li>
<li><p>STP - Statistical Parity</p></li>
<li><p>ACC - Accuracy</p></li>
<li><p>F1 - F1 Score</p></li>
<li><p>MCC - Matthews correlation coefficient</p></li>
</ul></li>
<li><p>groups_data - metrics across levels in protected variable</p></li>
<li><p>groups_confusion_matrices - confusion matrices for each subgroup</p></li>
<li><p>explainers - list of <code>DALEX</code> explainers used to create object</p></li>
<li><p>cutoffs - list of cutoffs for each explainer and subgroup</p></li>
<li><p>fairness_check_data - <code>data.frame</code> used for for plotting <code>fairness_object</code></p></li>
<li><p>... - other parameters passed to function</p></li>
</ul>

    <h2 class="hasAnchor" id="details"><a class="anchor" href="#details"></a>Details</h2>

    <p>Metrics used are made for each subgroup, then base metric score is subtracted leaving loss of particular metric.
If absolute loss is greater than epsilon than such metric is marked as "not passed". It means that values of metrics should be within (-epsilon,epsilon) boundary.
Epsilon value can be adjusted to user's needs. There are some metrics that might be derived from existing metrics (For example Equalized Odds - equal TPR and FPR for all subgroups).
That means passing 5 metrics in fairness check asserts that model is even more fair. In <code>fairness_check</code> models must always predict positive result. Not adhering to this rule
may lead to misinterpretation of the plot. More on metrics and their equivalents:
<a href='https://fairware.cs.umass.edu/papers/Verma.pdf'>https://fairware.cs.umass.edu/papers/Verma.pdf</a>
<a href='https://en.wikipedia.org/wiki/Fairness_(machine_learning)'>https://en.wikipedia.org/wiki/Fairness_(machine_learning)</a></p>
<p>Parity loss is computed as follows:
M_parity_loss = sum(abs(metric - base_metric))</p>
<p>where:</p>
<p>M - some metric mentioned above</p>
<p>metric - vector of metrics from each subgroup</p>
<p>base_metric - scalar, value of metric for base subgroup</p>
    <h2 class="hasAnchor" id="references"><a class="anchor" href="#references"></a>References</h2>

    <p>Zafar,Valera, Rodriguez, Gummadi (2017)  <a href='https://arxiv.org/pdf/1610.08452.pdf'>https://arxiv.org/pdf/1610.08452.pdf</a></p>
<p>Hardt, Price, Srebro (2016) <a href='https://arxiv.org/pdf/1610.02413.pdf'>https://arxiv.org/pdf/1610.02413.pdf</a></p>
<p>Verma, Rubin (2018) <a href='https://fairware.cs.umass.edu/papers/Verma.pdf'>https://fairware.cs.umass.edu/papers/Verma.pdf</a></p>

    <h2 class="hasAnchor" id="examples"><a class="anchor" href="#examples"></a>Examples</h2>
    <pre class="examples"><div class='input'><span class='fu'><a href='https://rdrr.io/r/utils/data.html'>data</a></span>(<span class='st'>"german"</span>)

<span class='no'>y_numeric</span> <span class='kw'>&lt;-</span> <span class='fu'><a href='https://rdrr.io/r/base/numeric.html'>as.numeric</a></span>(<span class='no'>german</span>$<span class='no'>Risk</span>) -<span class='fl'>1</span>

<span class='no'>lm_model</span> <span class='kw'>&lt;-</span> <span class='fu'><a href='https://rdrr.io/r/stats/glm.html'>glm</a></span>(<span class='no'>Risk</span>~<span class='no'>.</span>,
                <span class='kw'>data</span> <span class='kw'>=</span> <span class='no'>german</span>,
                <span class='kw'>family</span><span class='kw'>=</span><span class='fu'><a href='https://rdrr.io/r/stats/family.html'>binomial</a></span>(<span class='kw'>link</span><span class='kw'>=</span><span class='st'>"logit"</span>))

<span class='no'>rf_model</span> <span class='kw'>&lt;-</span> <span class='kw pkg'>ranger</span><span class='kw ns'>::</span><span class='fu'><a href='https://rdrr.io/pkg/ranger/man/ranger.html'>ranger</a></span>(<span class='no'>Risk</span> ~<span class='no'>.</span>,
                           <span class='kw'>data</span> <span class='kw'>=</span> <span class='no'>german</span>,
                           <span class='kw'>probability</span> <span class='kw'>=</span> <span class='fl'>TRUE</span>,
                           <span class='kw'>num.trees</span> <span class='kw'>=</span> <span class='fl'>200</span>)

<span class='no'>explainer_lm</span> <span class='kw'>&lt;-</span> <span class='kw pkg'>DALEX</span><span class='kw ns'>::</span><span class='fu'><a href='https://rdrr.io/pkg/DALEX/man/explain.html'>explain</a></span>(<span class='no'>lm_model</span>, <span class='kw'>data</span> <span class='kw'>=</span> <span class='no'>german</span>[,-<span class='fl'>1</span>], <span class='kw'>y</span> <span class='kw'>=</span> <span class='no'>y_numeric</span>)</div><div class='output co'>#&gt; Preparation of a new explainer is initiated
#&gt;   -&gt; model label       :  lm  ( <span style='color: #BBBB00;'> default </span><span> )
#&gt;   -&gt; data              :  1000  rows  9  cols 
#&gt;   -&gt; target variable   :  1000  values 
#&gt;   -&gt; predict function  :  yhat.glm  will be used ( </span><span style='color: #BBBB00;'> default </span><span> )
#&gt;   -&gt; predicted values  :  numerical, min =  0.1369187 , mean =  0.7 , max =  0.9832426  
#&gt;   -&gt; model_info        :  package stats , ver. 4.0.2 , task classification ( </span><span style='color: #BBBB00;'> default </span><span> ) 
#&gt;   -&gt; residual function :  difference between y and yhat ( </span><span style='color: #BBBB00;'> default </span><span> )
#&gt;   -&gt; residuals         :  numerical, min =  -0.9572803 , mean =  1.940006e-17 , max =  0.8283475  
#&gt;  </span><span style='color: #00BB00;'> A new explainer has been created! </span><span> </div><div class='input'><span class='no'>explainer_rf</span> <span class='kw'>&lt;-</span> <span class='kw pkg'>DALEX</span><span class='kw ns'>::</span><span class='fu'><a href='https://rdrr.io/pkg/DALEX/man/explain.html'>explain</a></span>(<span class='no'>rf_model</span>, <span class='kw'>data</span> <span class='kw'>=</span> <span class='no'>german</span>[,-<span class='fl'>1</span>], <span class='kw'>y</span> <span class='kw'>=</span> <span class='no'>y_numeric</span>)</div><div class='output co'>#&gt; Preparation of a new explainer is initiated
#&gt;   -&gt; model label       :  ranger  ( </span><span style='color: #BBBB00;'> default </span><span> )
#&gt;   -&gt; data              :  1000  rows  9  cols 
#&gt;   -&gt; target variable   :  1000  values 
#&gt;   -&gt; predict function  :  yhat.ranger  will be used ( </span><span style='color: #BBBB00;'> default </span><span> )
#&gt;   -&gt; predicted values  :  numerical, min =  0.0801369 , mean =  0.6975478 , max =  0.9979524  
#&gt;   -&gt; model_info        :  package ranger , ver. 0.12.1 , task classification ( </span><span style='color: #BBBB00;'> default </span><span> ) 
#&gt;   -&gt; residual function :  difference between y and yhat ( </span><span style='color: #BBBB00;'> default </span><span> )
#&gt;   -&gt; residuals         :  numerical, min =  -0.7406364 , mean =  0.002452163 , max =  0.6816389  
#&gt;  </span><span style='color: #00BB00;'> A new explainer has been created! </span><span> </div><div class='input'>
<span class='no'>fobject</span> <span class='kw'>&lt;-</span> <span class='fu'>fairness_check</span>(<span class='no'>explainer_lm</span>, <span class='no'>explainer_rf</span>,
                          <span class='kw'>protected</span> <span class='kw'>=</span> <span class='no'>german</span>$<span class='no'>Sex</span>,
                          <span class='kw'>privileged</span> <span class='kw'>=</span> <span class='st'>"male"</span>)</div><div class='output co'>#&gt; Creating fairness object
#&gt; -&gt; Privileged subgroup		: character (</span><span style='color: #00BB00;'> Ok </span><span> )
#&gt; -&gt; Protected variable		: factor (</span><span style='color: #00BB00;'> Ok </span><span> ) 
#&gt; -&gt; Cutoff values for explainers	: 0.5 ( for all subgroups ) 
#&gt; -&gt; Fairness objects		: 0 objects 
#&gt; -&gt; Checking explainers		: 2 in total ( </span><span style='color: #00BB00;'> compatible </span><span> )
#&gt; -&gt; Metric calculation		: 13/13 metrics calculated for all models
#&gt; </span><span style='color: #00BB00;'> Fairness object created succesfully </span><span> </div><div class='input'>
<span class='fu'><a href='https://rdrr.io/r/base/plot.html'>plot</a></span>(<span class='no'>fobject</span>)</div><div class='img'><img src='fairness_check-1.png' alt='' width='700' height='433' /></div><div class='input'>
</div></span></pre>
  </div>
  <div class="col-md-3 hidden-xs hidden-sm" id="pkgdown-sidebar">
    <nav id="toc" data-toggle="toc" class="sticky-top">
      <h2 data-toc-skip>Contents</h2>
    </nav>
  </div>
</div>


      <footer>
      <div class="copyright">
  <p>Developed by Jakub Wiśniewski, Przemysław Biecek.</p>
</div>

<div class="pkgdown">
  <p>Site built with <a href="https://pkgdown.r-lib.org/">pkgdown</a>.</p>
</div>

      </footer>
   </div>

  


  </body>
</html>


