examples/fair_checklists.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "4e81ede8",
   "metadata": {},
   "source": [
    "In this notebook, we create checklists that satisfy various fairness constraints on the UCI Adult dataset. Note that we use constraints that are quite loose due to time limitations. Using stricter constraints will require longer solution time, and could result in infeasibility."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "5c8e3e5b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using CPLEX version 20.1.0.0\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from IPChecklists.dataset import BinaryDataset\n",
    "\n",
    "# if using CPLEX\n",
    "from IPChecklists.model_cplex import ChecklistMIP\n",
    "from IPChecklists.constraints_cplex import (MaxNumFeatureConstraint, MConstraint, FNRConstraint, GroupFNRConstraint,  \n",
    "                                           GroupFNRGapConstraint, GroupFPRConstraint, GroupFPRGapConstraint)\n",
    "\n",
    "# if using Python-MIP\n",
    "# from IPChecklists.model_pythonmip import ChecklistMIP\n",
    "# from IPChecklists.constraints_pythonmip import (MaxNumFeatureConstraint, MConstraint, FNRConstraint, GroupFNRConstraint, \n",
    "#                                                 GroupFNRGapConstraint, GroupFPRConstraint, GroupFPRGapConstraint)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "7e76be12",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_datasets():\n",
    "    target = 'target'\n",
    "    pos_label = '>50K'\n",
    "    \n",
    "    df_train = pd.read_csv('./data/adult_train.csv', skipinitialspace = True)\n",
    "    df_test = pd.read_csv('./data/adult_test.csv', skipinitialspace = True)\n",
    "    df_test['target'] = df_test['target'].str.strip('.')\n",
    "    \n",
    "    for col in ['age', 'capital-gain', 'capital-loss', 'hours-per-week']: # continuous features\n",
    "        df_train[col] = df_train[col].astype(float)\n",
    "        df_test[col] = df_test[col].astype(float)\n",
    "\n",
    "    features = ['age', 'workclass', 'education', 'marital-status',\n",
    "               'occupation', 'relationship', 'capital-gain',\n",
    "               'capital-loss', 'hours-per-week']\n",
    "\n",
    "    df_train = df_train.sample(frac = 0.1, random_state = 42) # subsample for demonstration purposes for faster training\n",
    "        \n",
    "    train_ds = BinaryDataset(df_train, target_name = target, pos_label = pos_label, col_subset = features, \n",
    "                             add_complements=False, # don't add complements during binarization for faster training\n",
    "                            protected_attrs = ['sex', 'race'])\n",
    "    \n",
    "    test_ds = train_ds.apply_transform(df_test)\n",
    "    \n",
    "    return train_ds, test_ds"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9080e69c",
   "metadata": {},
   "source": [
    "### 1. Constraining the gap between groups"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7fb5d966",
   "metadata": {},
   "source": [
    "We create a checklist that minimizes 0-1 error (i.e. maximizes accuracy) with the following constraints:\n",
    "- N <= 5\n",
    "- M <= 3\n",
    "- |FPR(Male) - FPR(Female)| <= 5%\n",
    "- |FNR(Male) - FNR(Female)| <= 5%"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "b3bc28f2",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:root:Removed 3 non-informative columns: {'capital-gain>=0.0', 'occupation==?', 'capital-loss>=0.0'}\n",
      "INFO:root:Binary dataframe: 55 binary features and 3256 samples\n"
     ]
    }
   ],
   "source": [
    "train_ds, test_ds = get_datasets()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "136208e9",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = ChecklistMIP(train_ds, cost_func = '01', compress = False) # need to set compress=False for fairness constraints  \n",
    "model.add_constraint(GroupFNRGapConstraint('sex', eps = 0.05)) # FNR difference between all groups <= 5%\n",
    "model.add_constraint(GroupFPRGapConstraint('sex', eps = 0.05)) # FNR difference between all groups <= 5%\n",
    "model.build_problem(N_constraint = MaxNumFeatureConstraint('<=', 5), # N <= 5\n",
    "                       M_constraint= MConstraint('<=', 3), # M <= 3\n",
    "                     use_indicator = True # might result in better performance\n",
    "                   ) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "37925ef5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found solution with objective 716.0001689378772 and optimality gap 48.74%.\n"
     ]
    }
   ],
   "source": [
    "stats = model.solve(max_seconds=60, display_progress=False) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "2ebc284f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "workclass==Federal-gov\n",
       "education==Bachelors\n",
       "marital-status==Married-civ-spouse\n",
       "occupation==Exec-managerial\n",
       "relationship==Unmarried\n",
       "\n",
       "M = 2.0, N = 5.0"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "check = model.to_checklist()\n",
    "check"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "5c14ed08",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'accuracy': 0.7800982800982801,\n",
       " 'n_samples': 3256,\n",
       " 'TN': 2204,\n",
       " 'FN': 464,\n",
       " 'TP': 336,\n",
       " 'FP': 252,\n",
       " 'error': 716,\n",
       " 'TPR': 0.42,\n",
       " 'FNR': 0.58,\n",
       " 'FPR': 0.10260586319218241,\n",
       " 'TNR': 0.8973941368078175,\n",
       " 'precision': 0.5714285714285714,\n",
       " 'pred_prevalence': 0.18058968058968058,\n",
       " 'prevalence': 0.2457002457002457}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# training set performance\n",
    "check.get_metrics(train_ds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "f3a10281",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Female</th>\n",
       "      <th>Male</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>accuracy</th>\n",
       "      <td>0.867619</td>\n",
       "      <td>0.738441</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>n_samples</th>\n",
       "      <td>1050.000000</td>\n",
       "      <td>2206.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TN</th>\n",
       "      <td>870.000000</td>\n",
       "      <td>1334.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>FN</th>\n",
       "      <td>67.000000</td>\n",
       "      <td>397.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TP</th>\n",
       "      <td>41.000000</td>\n",
       "      <td>295.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>FP</th>\n",
       "      <td>72.000000</td>\n",
       "      <td>180.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>error</th>\n",
       "      <td>139.000000</td>\n",
       "      <td>577.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TPR</th>\n",
       "      <td>0.379630</td>\n",
       "      <td>0.426301</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>FNR</th>\n",
       "      <td>0.620370</td>\n",
       "      <td>0.573699</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>FPR</th>\n",
       "      <td>0.076433</td>\n",
       "      <td>0.118890</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TNR</th>\n",
       "      <td>0.923567</td>\n",
       "      <td>0.881110</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>precision</th>\n",
       "      <td>0.362832</td>\n",
       "      <td>0.621053</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pred_prevalence</th>\n",
       "      <td>0.107619</td>\n",
       "      <td>0.215322</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>prevalence</th>\n",
       "      <td>0.102857</td>\n",
       "      <td>0.313690</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      Female         Male\n",
       "accuracy            0.867619     0.738441\n",
       "n_samples        1050.000000  2206.000000\n",
       "TN                870.000000  1334.000000\n",
       "FN                 67.000000   397.000000\n",
       "TP                 41.000000   295.000000\n",
       "FP                 72.000000   180.000000\n",
       "error             139.000000   577.000000\n",
       "TPR                 0.379630     0.426301\n",
       "FNR                 0.620370     0.573699\n",
       "FPR                 0.076433     0.118890\n",
       "TNR                 0.923567     0.881110\n",
       "precision           0.362832     0.621053\n",
       "pred_prevalence     0.107619     0.215322\n",
       "prevalence          0.102857     0.313690"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# training set fairness, note that the two constraints are satisfied\n",
    "check.get_fairness_metrics(train_ds, attributes=['sex']) "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c220ff60",
   "metadata": {},
   "source": [
    "### 2. Constraining the per-group worst performance"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b0135b9e",
   "metadata": {},
   "source": [
    "We create a checklist that minimizes 0-1 error (i.e. maximizes accuracy) with the following constraints:\n",
    "- N <= 8\n",
    "- M <= 4\n",
    "- max{FPR(Male), FPR(Female)} <= 35%\n",
    "- max{FNR(Male), FNR(Female)} <= 35%"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "5ab52d68",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:root:Removed 3 non-informative columns: {'capital-gain>=0.0', 'occupation==?', 'capital-loss>=0.0'}\n",
      "INFO:root:Binary dataframe: 55 binary features and 3256 samples\n"
     ]
    }
   ],
   "source": [
    "train_ds, test_ds = get_datasets()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "ac033505",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = ChecklistMIP(train_ds, cost_func = '01', compress = False) # need to set compress=False for fairness constraints  \n",
    "model.add_constraint(GroupFNRConstraint('sex', eps = 0.35)) \n",
    "model.add_constraint(GroupFPRConstraint('sex', eps = 0.35)) \n",
    "model.build_problem(N_constraint = MaxNumFeatureConstraint('<=', 8), # N <= 5\n",
    "                       M_constraint= MConstraint('<=', 4), # M <= 3\n",
    "                    use_indicator = True\n",
    "                   ) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "57ccf193",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Advanced basis not built.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found solution with objective 783.0002371974247 and optimality gap 56.23%.\n"
     ]
    }
   ],
   "source": [
    "stats = model.solve(max_seconds=60, display_progress=False) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "71a34f60",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "age>=37.0\n",
       "workclass==Self-emp-not-inc\n",
       "education==Prof-school\n",
       "marital-status==Married-civ-spouse\n",
       "occupation==Exec-managerial\n",
       "relationship==Wife\n",
       "hours-per-week>=40.0\n",
       "\n",
       "M = 3.0, N = 7.0"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "check = model.to_checklist()\n",
    "check"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "113594fc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Female</th>\n",
       "      <th>Male</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>accuracy</th>\n",
       "      <td>0.851429</td>\n",
       "      <td>0.715775</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>n_samples</th>\n",
       "      <td>1050.000000</td>\n",
       "      <td>2206.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TN</th>\n",
       "      <td>822.000000</td>\n",
       "      <td>1052.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>FN</th>\n",
       "      <td>36.000000</td>\n",
       "      <td>165.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TP</th>\n",
       "      <td>72.000000</td>\n",
       "      <td>527.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>FP</th>\n",
       "      <td>120.000000</td>\n",
       "      <td>462.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>error</th>\n",
       "      <td>156.000000</td>\n",
       "      <td>627.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TPR</th>\n",
       "      <td>0.666667</td>\n",
       "      <td>0.761561</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>FNR</th>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.238439</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>FPR</th>\n",
       "      <td>0.127389</td>\n",
       "      <td>0.305152</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TNR</th>\n",
       "      <td>0.872611</td>\n",
       "      <td>0.694848</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>precision</th>\n",
       "      <td>0.375000</td>\n",
       "      <td>0.532861</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pred_prevalence</th>\n",
       "      <td>0.182857</td>\n",
       "      <td>0.448323</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>prevalence</th>\n",
       "      <td>0.102857</td>\n",
       "      <td>0.313690</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      Female         Male\n",
       "accuracy            0.851429     0.715775\n",
       "n_samples        1050.000000  2206.000000\n",
       "TN                822.000000  1052.000000\n",
       "FN                 36.000000   165.000000\n",
       "TP                 72.000000   527.000000\n",
       "FP                120.000000   462.000000\n",
       "error             156.000000   627.000000\n",
       "TPR                 0.666667     0.761561\n",
       "FNR                 0.333333     0.238439\n",
       "FPR                 0.127389     0.305152\n",
       "TNR                 0.872611     0.694848\n",
       "precision           0.375000     0.532861\n",
       "pred_prevalence     0.182857     0.448323\n",
       "prevalence          0.102857     0.313690"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# training set fairness, note that the two constraints are satisfied\n",
    "check.get_fairness_metrics(train_ds, attributes=['sex']) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c27ae320",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}