XGBoost F: all mitigation techniques.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import config as cfg\n",
    "import pandas as pd\n",
    "from tools import file_reader, file_writer, explainer\n",
    "from utility import metrics\n",
    "from sklearn.metrics import accuracy_score, precision_score\n",
    "from sklearn.metrics import recall_score, roc_auc_score\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.model_selection import train_test_split\n",
    "import xgboost as xgb\n",
    "import os\n",
    "from sklearn.metrics import confusion_matrix\n",
    "import seaborn as sns\n",
    "from sklearn import preprocessing\n",
    "\n",
    "from sklearn.model_selection import KFold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from utils_Copy import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def LFR_custom(df_train,y_train,lfr=None):\n",
    "    from aif360.algorithms.preprocessing import LFR\n",
    "    from aif360.datasets import BinaryLabelDataset\n",
    "    \n",
    "    df_train=pd.concat([df_train,y_train],axis=1)\n",
    "    \n",
    "    X_col_names_f=['Gender', 'BirthYear', 'LoanPeriod', 'NumberAts']\n",
    "    df2_all=df_train.drop(columns=X_col_names_f).copy() #Saves all columns, except numerical og gender\n",
    "    df2=df_train[X_col_names_f+[\"Fall\"]].copy() #Saves only numerical features\n",
    "    df2_gender=df_train[\"Gender\"].copy() #Saves only gender\n",
    "    \n",
    "    \n",
    "    #Create the binarylabeldataset\n",
    "    df_BLD = BinaryLabelDataset(favorable_label='1',\n",
    "                                unfavorable_label='0',\n",
    "                                df=df2,\n",
    "                                label_names=['Fall'],\n",
    "                                protected_attribute_names=[\"Gender\"],\n",
    "                                unprivileged_protected_attributes=['0'])\n",
    "    #Define the DI remover\n",
    "    if lfr is None:\n",
    "        lfr = LFR(privileged_groups=[{\"Gender\": 1}], \n",
    "                                    unprivileged_groups=[{\"Gender\": 0}]\n",
    "                  \n",
    "                 )\n",
    "        rp_df = lfr.fit_transform(df_BLD)\n",
    "    else:\n",
    "        rp_df = lfr.transform(df_BLD)\n",
    "        \n",
    "\n",
    "    #Save the columnnames\n",
    "    all_col_names=df_BLD.feature_names+df_BLD.label_names\n",
    "        \n",
    "    #Save repaired data as pandas DF\n",
    "    rp_df_pd = pd.DataFrame(np.hstack([rp_df.features,rp_df.labels]),columns=all_col_names) \n",
    "    \n",
    "    # OBS: Somehow gender is also transformed! So we drop it! \n",
    "    rp_df_pd = rp_df_pd.drop(columns=[\"Gender\"])\n",
    "    \n",
    "    #Concatenate the non-numerical columns\n",
    "    transformed_data = pd.concat ([rp_df_pd,df2_all], axis=1)\n",
    "    \n",
    "    \n",
    "    transformed_data=transformed_data.drop(columns=[\"Fall\"])\n",
    "    \n",
    "    return transformed_data,lfr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def DI_remove_custom(df_train,RP_level=1.0,drop_d=False,y_train=None):\n",
    "    from aif360.algorithms.preprocessing import DisparateImpactRemover\n",
    "    from aif360.datasets import BinaryLabelDataset\n",
    "    \n",
    "    if drop_d:\n",
    "        df_train=pd.concat([df_train,y_train],axis=1)\n",
    "    \n",
    "    X_col_names_f=['Gender', 'BirthYear', 'LoanPeriod', 'NumberAts']\n",
    "    df2_all=df_train.drop(columns=X_col_names_f).copy() #Saves all columns, except numerical og gender\n",
    "    df2=df_train[X_col_names_f].copy() #Saves only numerical features\n",
    "    \n",
    "    df2[\"dummy\"]=1 # this is a dummy variable, since DI remover dont use y. \n",
    "    \n",
    "    #Create the binarylabeldataset\n",
    "    df_BLD = BinaryLabelDataset(favorable_label='1',\n",
    "                                unfavorable_label='0',\n",
    "                                df=df2,\n",
    "                                label_names=['dummy'],\n",
    "                                protected_attribute_names=[\"Gender\"],\n",
    "                                unprivileged_protected_attributes=['0'])\n",
    "    #Define the DI remover\n",
    "    di = DisparateImpactRemover(repair_level=RP_level)\n",
    "    #Save the columnnames\n",
    "    all_col_names=df_BLD.feature_names+df_BLD.label_names\n",
    "    #Reparing the data\n",
    "    rp_df = di.fit_transform(df_BLD)  \n",
    "    #Save repaired data as pandas DF\n",
    "    rp_df_pd = pd.DataFrame(np.hstack([rp_df.features,rp_df.labels]),columns=all_col_names) \n",
    "    #Concatenate the non-numerical columns\n",
    "    transformed_data = pd.concat ([rp_df_pd,df2_all], axis=1)\n",
    "    \n",
    "    transformed_data_train=transformed_data.drop(columns=[\"dummy\"])\n",
    "    \n",
    "    if drop_d:\n",
    "        transformed_data_train=transformed_data_train.drop(columns=[\"Gender\"])\n",
    "\n",
    "    \n",
    "    return transformed_data_train\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "procted_col_name=\"Gender\"\n",
    "target_name = \"Fall\"\n",
    "y_col_name=target_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pathRoot=\"../../Data_air/\"\n",
    "pathFall=pathRoot+\"Fall_count_clusterOHE_std.csv\"\n",
    "df=pd.read_csv(pathFall)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "titel_mitigation=\"DI remove no gender\" #example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dropping_D=True\n",
    "gender_swap=False\n",
    "DI_remove=True\n",
    "LFR_mitigation=False #Set dropping_D=True, men without removing it from X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "PATH_orig=\"../Xgboost/models/\"+titel_mitigation+\"/\"\n",
    "PATH=PATH_orig\n",
    "\n",
    "\n",
    "#Make dir to files\n",
    "if not os.path.exists(PATH):\n",
    "    os.makedirs(PATH)\n",
    "    print(\"Created new path!: \",PATH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "X_col_names=[\n",
    "#'Gender',\n",
    "'BirthYear',\n",
    "'LoanPeriod',\n",
    "'NumberAts',\n",
    "'Ats_Polstring',\n",
    "'Ats_Mobilitystokke',\n",
    "'Ats_Belysning',\n",
    "'Ats_Underlag',\n",
    "'Ats_ToiletforhøjereStativ',\n",
    "'Ats_Signalgivere',\n",
    "'Ats_EldrevneKørestole',\n",
    "'Ats_Forstørrelsesglas',\n",
    "'Ats_Nødalarmsystemer',\n",
    "'Ats_MobilePersonløftere',\n",
    "'Ats_TrappelifteMedPlatforme',\n",
    "'Ats_Badekarsbrætter',\n",
    "'Ats_Albuestokke',\n",
    "'Ats_MaterialerOgRedskaberTilAfmærkning',\n",
    "'Ats_Ryglæn',\n",
    "#'Ats_0',\n",
    "'Ats_GanghjælpemidlerStøtteTilbehør',\n",
    "'Ats_Støttebøjler',\n",
    "'Ats_Lejringspuder',\n",
    "'Ats_Strømpepåtagere',\n",
    "'Ats_Dørtrin',\n",
    "'Ats_Spil',\n",
    "'Ats_BordePåStole',\n",
    "'Ats_Drejeskiver',\n",
    "'Ats_Toiletstole',\n",
    "'Ats_LøftereStationære',\n",
    "'Ats_Madmålingshjælpemidler',\n",
    "'Ats_Fodbeskyttelse',\n",
    "'Ats_Ståløftere',\n",
    "'Ats_Stole',\n",
    "'Ats_Sengeborde',\n",
    "'Ats_Toiletter',\n",
    "'Ats_ToiletforhøjereFaste',\n",
    "'Ats_Påklædning',\n",
    "'Ats_Brusere',\n",
    "'Ats_VævsskadeLiggende',\n",
    "'Ats_Døråbnere',\n",
    "'Ats_ServeringAfMad',\n",
    "'Ats_TrappelifteMedSæder',\n",
    "'Ats_SæderTilMotorkøretøjer',\n",
    "'Ats_KørestoleManuelleHjælper',\n",
    "'Ats_Gangbukke',\n",
    "'Ats_Rollatorer',\n",
    "'Ats_TryksårsforebyggendeSidde',\n",
    "'Ats_Fastnettelefoner',\n",
    "'Ats_Bækkener',\n",
    "'Ats_Vendehjælpemidler',\n",
    "'Ats_Sanseintegration',\n",
    "'Ats_Kørestolsbeskyttere',\n",
    "'Ats_Arbejdsstole',\n",
    "'Ats_Løftesejl',\n",
    "'Ats_KørestoleForbrændingsmotor',\n",
    "'Ats_Løftestropper',\n",
    "'Ats_Stiger',\n",
    "'Ats_TransportTrapper',\n",
    "'Ats_DrivaggregaterKørestole',\n",
    "'Ats_Emballageåbnere',\n",
    "'Ats_ToiletforhøjereLøse',\n",
    "'Ats_Hårvask',\n",
    "'Ats_PersonløftereStationære',\n",
    "'Ats_Madrasser',\n",
    "'Ats_Vinduesåbnere',\n",
    "'Ats_Læsestativer',\n",
    "'Ats_KørestoleManuelleDrivringe',\n",
    "'Ats_Sædepuder',\n",
    "'Ats_UdstyrCykler',\n",
    "'Ats_Karkludsvridere',\n",
    "'Ats_Vaskeklude',\n",
    "'Ats_Sengeudstyr',\n",
    "'Ats_Madlavningshjælpemidler',\n",
    "'Ats_Skohorn',\n",
    "'Ats_GribetængerManuelle',\n",
    "'Ats_Hvilestole',\n",
    "'Ats_EldrevneKørestoleStyring',\n",
    "'Ats_BærehjælpemidlerTilKørestole',\n",
    "'Ats_LøftegalgerSeng',\n",
    "'Ats_Høreforstærkere',\n",
    "'Ats_Kalendere',\n",
    "'Ats_Stokke',\n",
    "'Ats_Løftegalger',\n",
    "'Ats_Ure',\n",
    "'Ats_StøttegrebFlytbare',\n",
    "'Ats_Forflytningsplatforme',\n",
    "'Ats_RamperFaste',\n",
    "'Ats_Rygehjælpemidler',\n",
    "'Ats_Personvægte',\n",
    "'Ats_Manøvreringshjælpemidler',\n",
    "'Ats_Overtøj',\n",
    "'Ats_Lydoptagelse',\n",
    "'Ats_Gangborde',\n",
    "'Ats_Ståstøttestole',\n",
    "'Ats_RamperMobile',\n",
    "'Ats_Bærehjælpemidler',\n",
    "'Ats_Badekarssæder',\n",
    "'Ats_Siddemodulsystemer',\n",
    "'Ats_Videosystemer',\n",
    "'Ats_Siddepuder',\n",
    "'Ats_Sengeheste',\n",
    "'Ats_Stolerygge',\n",
    "'Ats_Rulleborde',\n",
    "'Ats_Sengeforlængere',\n",
    "'Ats_Madningsudstyr',\n",
    "'Ats_Brusestole',\n",
    "'Ats_Flerpunktsstokke',\n",
    "'Ats_SengebundeMedMotor',\n",
    "'Ats_Cykler',\n",
    "'Ats_CykelenhederKørestole',\n",
    "'Ats_Stokkeholdere',\n",
    "'Ats_Toiletarmstøtter',\n",
    "'Ats_Coxitstole',\n",
    "'Ats_Toiletsæder',\n",
    "'Ats_Rebstiger',\n",
    "'Ats_Forhøjerklodser',\n",
    "'Cluster_0',\n",
    "'Cluster_1',\n",
    "'Cluster_2',\n",
    "'Cluster_3',\n",
    "'Cluster_4',\n",
    "'Cluster_5',\n",
    "'Cluster_6',\n",
    "'Cluster_7',\n",
    "'Cluster_8',\n",
    "'Cluster_9',\n",
    "'Cluster_10',\n",
    "'Cluster_11',\n",
    "'Cluster_12',\n",
    "'Cluster_13',\n",
    "'Cluster_14',\n",
    "'Cluster_15',\n",
    "'Cluster_16',\n",
    "'Cluster_17',\n",
    "'Cluster_18',\n",
    "'Cluster_19']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "modelcounter=0\n",
    "df_test=pd.DataFrame([],columns=list(X_col_names)+[\"Fall\"]+[\"output\"]+[\"output_prob\"])\n",
    "\n",
    "if LFR_mitigation==True:\n",
    "    df_test.drop(columns=[\"Gender\"])\n",
    "\n",
    "for new_seed in range(1,11):\n",
    "    df = df.sample(frac=1, random_state=new_seed).reset_index(drop=True)\n",
    "      \n",
    "    X = df[X_col_names]\n",
    "    y = df[target_name].to_frame()\n",
    "    \n",
    "    neg, pos = np.bincount(y[target_name])\n",
    "    scale_pos_weight = neg / pos\n",
    "\n",
    "    params = {\"n_estimators\": 400,\n",
    "            \"objective\": \"binary:logistic\",\n",
    "            \"scale_pos_weight\": scale_pos_weight,\n",
    "            \"use_label_encoder\": False,\n",
    "            \"learning_rate\": 0.1,\n",
    "            \"eval_metric\": \"logloss\",\n",
    "            \"seed\": 0\n",
    "    }\n",
    "    \n",
    "    \n",
    "    model = xgb.XGBClassifier(**params)\n",
    "    #skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=new_seed)\n",
    "    skf=KFold(n_splits=5, random_state=new_seed, shuffle=True)\n",
    "\n",
    "    \n",
    "    y_valid_pred = 0*y[target_name]\n",
    "    valid_acc, valid_pre, valid_recall, valid_roc_auc = list(), list(), list(), list()\n",
    "    #for train_index, valid_index in skf.split(X_train, y_train):\n",
    "    for train_index, valid_index in skf.split(X):\n",
    "        print(f\"Running model {modelcounter}\")\n",
    "\n",
    "        X_train_split, X_valid_split = X.iloc[train_index,:], X.iloc[valid_index,:]\n",
    "        y_train_split, y_valid_split = y.iloc[train_index], y.iloc[valid_index]\n",
    "        \n",
    "        if gender_swap==True:\n",
    "            X_train_split_copy=X_train_split.copy()\n",
    "            y_train_split_copy=y_train_split.copy()\n",
    "            \n",
    "            X_train_split_copy[\"Gender\"]=(X_train_split_copy[\"Gender\"]-1)*(-1)\n",
    "            \n",
    "            X_train_split=pd.concat([X_train_split,X_train_split_copy])\n",
    "            \n",
    "            y_train_split=pd.concat([y_train_split,y_train_split_copy])\n",
    "            \n",
    "        if DI_remove==True:\n",
    "            X_train_split=DI_remove_custom(X_train_split.reset_index(drop=True),drop_d=dropping_D,y_train=df[procted_col_name].iloc[train_index].to_frame().reset_index(drop=True))\n",
    "            X_valid_split=DI_remove_custom(X_valid_split.reset_index(drop=True),drop_d=dropping_D,y_train=df[procted_col_name].iloc[valid_index].to_frame().reset_index(drop=True))\n",
    "            \n",
    "        if LFR_mitigation==True:\n",
    "            X_train_split,lfr=LFR_custom(X_train_split.reset_index(drop=True),\n",
    "                                         y_train_split.reset_index(drop=True),\n",
    "                                         lfr=None)\n",
    "            X_valid_split,lfr=LFR_custom(X_valid_split.reset_index(drop=True),\n",
    "                                         y_valid_split.reset_index(drop=True),\n",
    "                                         lfr)\n",
    "            \n",
    "        \n",
    "        optimize_rounds = True\n",
    "        early_stopping_rounds = 50\n",
    "        if optimize_rounds:\n",
    "            eval_set=[(X_valid_split, y_valid_split)]\n",
    "            fit_model = model.fit(X_train_split, y_train_split, \n",
    "                                    eval_set=eval_set,\n",
    "                                    eval_metric=metrics.gini_xgb,\n",
    "                                    early_stopping_rounds=early_stopping_rounds,\n",
    "                                    verbose=False)\n",
    "\n",
    "        else:\n",
    "            fit_model = model.fit(X_train_split, y_train_split)\n",
    "\n",
    "        pred = fit_model.predict_proba(X_valid_split)[:,1]\n",
    "        y_valid_pred.iloc[valid_index] = pred\n",
    "\n",
    "        y_valid_scores = (y_valid_pred.iloc[valid_index] > 0.5)\n",
    "\n",
    "\n",
    "        #### SAVE DATA####\n",
    "        y_true_pd=y_valid_split.reset_index(drop=True)\n",
    "        y_pred_pd=y_valid_scores.apply(lambda x: 1 if x==True else 0).to_frame().reset_index(drop=True).rename(columns={\"Fall\":\"output\"})\n",
    "        y_pred_prob_pd=pd.DataFrame(pred, columns = [\"output_prob\"])\n",
    "        \n",
    "\n",
    "        df_subset=pd.concat([X_valid_split.reset_index(drop=True),y_true_pd,y_pred_pd,y_pred_prob_pd],axis=1)\n",
    "        \n",
    "        \n",
    "        if dropping_D==True:\n",
    "            df_subset[procted_col_name]=list(df[procted_col_name].iloc[valid_index])\n",
    "        \n",
    "        df_subset[\"Model\"]=\"Model\"+str(modelcounter)\n",
    "        \n",
    "        df_subset.to_csv(PATH+\"model\"+str(modelcounter)+\"_test_data.csv\")\n",
    "\n",
    "        df_test=df_test.append(df_subset, ignore_index=True)\n",
    "\n",
    "        valid_acc.append(accuracy_score(y_valid_split, y_valid_scores))\n",
    "        valid_pre.append(precision_score(y_valid_split, y_valid_scores))\n",
    "        valid_recall.append(recall_score(y_valid_split, y_valid_scores))\n",
    "        valid_roc_auc.append(roc_auc_score(y_valid_split, y_valid_pred.iloc[valid_index]))\n",
    "\n",
    "        modelcounter=modelcounter+1\n",
    "\n",
    "df_test.to_csv(PATH+\"all_test_data.csv\")\n",
    "print(\"The full test data lies here:\",PATH+\"all_test_data.csv\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}