Final Reject Inference Notebook.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Reject Inference Compiled Notebook #"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Packages ###"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from numpy import mean\n",
    "from numpy import std\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import joblib\n",
    "\n",
    "from sklearn import tree\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.impute import KNNImputer, SimpleImputer\n",
    "from sklearn.model_selection import GridSearchCV, cross_val_score, RandomizedSearchCV\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import recall_score, confusion_matrix, roc_auc_score\n",
    "\n",
    "from imblearn.ensemble import BalancedRandomForestClassifier\n",
    "\n",
    "from pgmpy.estimators import ExhaustiveSearch, BicScore, BayesianEstimator\n",
    "from pgmpy.models import BayesianNetwork\n",
    "\n",
    "import RIfunctions as ri\n",
    "\n",
    "from dataprep.eda import create_report"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Variables ###"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "RANDOM_STATE = 123\n",
    "CROSS_VALIDATION = 10\n",
    "results_df = pd.DataFrame()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read CSV files into pandas dataframe\n",
    "application_df = pd.read_csv('Data\\input_file_1.csv', sep = ',', index_col=0)\n",
    "accept_df = pd.read_csv('Data\\input_file_2.csv', sep = ',', index_col=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "create_report(accept_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Need to split the dataset early for a consistent test dataset\n",
    "# Removing instances from the application dataset so there is no knowledge\n",
    "\n",
    "'''Exclude the period from the beginning of 2016 onward as it is noticed in time-series plots that this period has\n",
    " a significant portion of the expected defaults not reported yet. This would hence be biased.'''\n",
    "\n",
    "accept_df['issue_d'] = pd.to_datetime(accept_df['issue_d'])\n",
    "application_df['issue_d'] = pd.to_datetime(application_df['issue_d'])\n",
    "\n",
    "print(accept_df.shape)\n",
    "\n",
    "application_df = application_df.loc[application_df['issue_d'] < '2016-1-1']\n",
    "accept_df = accept_df.loc[accept_df['issue_d'] < '2016-1-1']\n",
    "\n",
    "accept_train_df = accept_df.loc[accept_df['issue_d'] < accept_df['issue_d'].quantile(0.8)]\n",
    "accept_test_df = accept_df.loc[accept_df['issue_d'] >= accept_df['issue_d'].quantile(0.8)]\n",
    "\n",
    "application_df = application_df.loc[application_df['issue_d'] <= max(accept_train_df[\"issue_d\"])]\n",
    "\n",
    "application_df = application_df.drop('issue_d', axis=1)\n",
    "accept_test_df = accept_test_df.drop('issue_d', axis=1)\n",
    "accept_train_df = accept_train_df.drop('issue_d', axis=1)\n",
    "\n",
    "print(accept_df.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Application Dataset ##"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Pre-processing ###"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Remove instances with a small number of purpose values and align all values\n",
    "\n",
    "application_df = ri.purposeCleaning(application_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "corr = application_df.corr()\n",
    "\n",
    "#setting the size of plot\n",
    "fig = plt.figure(figsize = (12, 9))\n",
    "#masking the upper traingle part since matrix is symmetric(repetitive) \n",
    "mask = np.triu(np.ones_like(corr, dtype=bool))\n",
    "sns.heatmap(corr, vmax = .8,mask=mask, square = True, annot = True)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get dummies for the purpose variable as it is an object which cannot be interperted   \n",
    "\n",
    "application_df = pd.get_dummies(application_df, columns=['purpose'], drop_first=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Train the model on the full dataset. We are trying to build a pre-existing, trained model, not a new model.\n",
    "\n",
    "y_train = application_df['rejected']\n",
    "X_train = application_df.drop(['rejected'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "total, pos, rate = ri.splitCount(application_df['rejected'])\n",
    "\n",
    "print('Training Set:\\n    Total: {}\\n    Positive: {} ({:.2f}% of total)\\n'.format(\n",
    "    total, pos, rate))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Logistic Regression Model for Application Dataset ###"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # # Optimisation\n",
    "\n",
    "# pipeline_sgdlogreg = Pipeline([\n",
    "#     # Alternative is KNNImputer\n",
    "#     ('imputer', SimpleImputer(copy=False)), # Mean imputation by default\n",
    "#     ('scaler', StandardScaler(copy=False)),\n",
    "#     ('model',LogisticRegression(\n",
    "#         class_weight='balanced',\n",
    "#         random_state=RANDOM_STATE,\n",
    "#         n_jobs=-2,\n",
    "#         warm_start=True\n",
    "#         )\n",
    "#     )\n",
    "# ])\n",
    "\n",
    "# param_grid_sgdlogreg = {\n",
    "#     'model__solver': ['newton-cg', 'lbfgs', 'sag', 'saga', 'liblinear']\n",
    "# }\n",
    "\n",
    "# grid_sgdlogreg = GridSearchCV(\n",
    "#     estimator=pipeline_sgdlogreg,\n",
    "#     param_grid=param_grid_sgdlogreg,\n",
    "#     scoring='roc_auc',\n",
    "#     pre_dispatch=3,\n",
    "#     n_jobs=-2,\n",
    "#     cv=CROSS_VALIDATION,\n",
    "#     verbose=1,\n",
    "#     return_train_score=False\n",
    "# )\n",
    "\n",
    "# grid_sgdlogreg.fit(X_train, y_train)\n",
    "\n",
    "# print(grid_sgdlogreg.best_estimator_,grid_sgdlogreg.best_params_, grid_sgdlogreg.best_score_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Hyperparemeters already optimised\n",
    "\n",
    "pipeline_logreg = Pipeline([\n",
    "    ('imputer', SimpleImputer(copy=False)),\n",
    "    ('scaler', StandardScaler(copy=False)),\n",
    "    ('model', LogisticRegression(\n",
    "        class_weight='balanced',\n",
    "        max_iter=1000, \n",
    "        tol = 1e-3,\n",
    "        random_state=RANDOM_STATE,\n",
    "        n_jobs=-1,\n",
    "        warm_start=True,\n",
    "        solver = 'lbfgs'\n",
    "        )\n",
    "    )\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pipeline_logreg.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cv_scores_lr = cross_val_score(estimator= pipeline_logreg, X=X_train, y=y_train, scoring='roc_auc', cv=CROSS_VALIDATION, n_jobs=-1)\n",
    "print(\"The cross-validation AUC for optimised LR model after a 10 fold cross validation: %.3f (%.3f)\" % (mean(cv_scores_lr), std(cv_scores_lr)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Scores of each application and model performance\n",
    "\n",
    "y_score = pipeline_logreg.predict_proba(X_train)[:,1]\n",
    "\n",
    "y_score_flag = [int(round(i)) for i in y_score]\n",
    "\n",
    "print(roc_auc_score(y_train, y_score), recall_score(y_train, y_score_flag, pos_label=1), recall_score(y_train, y_score_flag, pos_label=0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Computing Confusion matrix for the above algorithm\n",
    "\n",
    "cnf_matrix = confusion_matrix(y_train, y_score_flag)\n",
    "np.set_printoptions(precision=2)\n",
    "\n",
    "# Plot non-normalized confusion matrix\n",
    "plt.figure()\n",
    "ri.plot_confusion_matrix(cnf_matrix, classes=[\"ACCEPT\", \"REJECT\"])\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_df = ri.AppendResults(df=results_df, title=\"LR Application Dataset\",\n",
    "AUC_scores=cv_scores_lr, y_test=y_train, y_score=y_score)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Save LR Results ###"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Add score column to the dataframe\n",
    "\n",
    "application_df['score'] = np.hstack(y_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Undummy the purpose column\n",
    "\n",
    "def undummy(d):\n",
    "    return d.dot(d.columns)\n",
    "\n",
    "application_df = application_df.assign(purpose=application_df.filter(regex='^purpose').pipe(undummy))\n",
    "application_df['purpose'] = application_df['purpose'].str.replace(\"purpose_\",\"\")\n",
    "application_df = application_df[application_df.columns.drop(list(application_df.filter(regex=\"purpose_\")))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save to CSV file\n",
    "\n",
    "application_df.to_csv('AR Scores.csv', mode='w+')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Accepted Loans Dataset ##"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Pre-processing ###"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rows with Educational and Renewable Energy purpose have been dropped in the application dataset\n",
    "\n",
    "accept_train_df = accept_train_df.drop(accept_train_df[accept_train_df['purpose'] == 'educational'].index)\n",
    "accept_test_df = accept_test_df.drop(accept_test_df[accept_test_df['purpose'] == 'educational'].index)\n",
    "\n",
    "accept_train_df = accept_train_df.drop(accept_train_df[accept_train_df['purpose'] == 'renewable_energy'].index)\n",
    "accept_test_df = accept_test_df.drop(accept_test_df[accept_test_df['purpose'] == 'renewable_energy'].index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_cols = list(accept_df.columns)\n",
    "all_cols.remove('charged_off')\n",
    "all_cols.remove('issue_d')\n",
    "\n",
    "to_drop_categorical = ['home_ownership', 'verification_status', 'purpose', 'application_type']\n",
    "\n",
    "for i in to_drop_categorical:\n",
    "\n",
    "    all_cols.remove(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "corr = accept_df.corr()\n",
    "\n",
    "#setting the size of plot\n",
    "fig = plt.figure(figsize = (12, 9))\n",
    "#masking the upper traingle part since matrix is symmetric(repetitive) \n",
    "mask = np.triu(np.ones_like(corr, dtype=bool))\n",
    "sns.heatmap(corr, vmax = .8,mask=mask, square = True, annot = True)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Application value did not exist in the training set but did in the test\n",
    "\n",
    "accept_train_df[\"at_Joint App\"] = 0\n",
    "\n",
    "accept_train_df_all = pd.get_dummies(accept_train_df,prefix=[\"ho\",\"vs\",\"purp\",\"at\"], columns=['home_ownership', 'verification_status', 'purpose', 'application_type'], drop_first=False)\n",
    "accept_test_df_all = pd.get_dummies(accept_test_df,prefix=[\"ho\",\"vs\",\"purp\",\"at\"], columns=['home_ownership', 'verification_status', 'purpose', 'application_type'], drop_first=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "scaler_all = StandardScaler(copy=False)\n",
    "\n",
    "accept_train_df_all[all_cols] = accept_train_df_all[all_cols].fillna(accept_train_df_all[all_cols].mean())\n",
    "accept_test_df_all[all_cols] = accept_test_df_all[all_cols].fillna(accept_test_df_all[all_cols].mean())\n",
    "\n",
    "accept_train_df_all[all_cols] = scaler_all.fit_transform(accept_train_df_all[all_cols], accept_train_df_all['charged_off'])\n",
    "accept_test_df_all[all_cols] = scaler_all.transform(accept_test_df_all[all_cols])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "total, pos, rate = ri.splitCount(accept_train_df_all['charged_off'])\n",
    "print('Training Set:\\n    Total: {}\\n    Positive: {} ({:.2f}% of total)\\n'.format(\n",
    "    total, pos, rate))\n",
    "\n",
    "total, pos, rate = ri.splitCount(accept_test_df_all['charged_off'])\n",
    "print('Testing Set:\\n    Total: {}\\n    Positive: {} ({:.2f}% of total)\\n'.format(\n",
    "    total, pos, rate))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = accept_train_df_all['charged_off']\n",
    "y_test = accept_test_df_all['charged_off']\n",
    "\n",
    "X_train = accept_train_df_all.drop('charged_off', axis=1)\n",
    "X_test = accept_test_df_all.drop('charged_off', axis=1)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Random Forest Model ###"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Hyperparameter tuning. Took over 11 hours to run.\n",
    "\n",
    "# best_hyperparams, best_rf = ri.RandomForestTuning(BalancedRandomForestClassifier(), X_train, y_train, filename = \"./random_forest.joblib\", cv = CROSS_VALIDATION)\n",
    "# print(\"Best hyperparameters: \\n\", best_hyperparams )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = joblib.load(\"Tuned Models/random_forest.joblib\")\n",
    "print(model.get_params())\n",
    "model.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cv_scores = cross_val_score(estimator= model, X=X_train, y=y_train, scoring='roc_auc', cv=CROSS_VALIDATION, n_jobs=-1)\n",
    "print(\"(Training Set) The cross-validation AUC for optimised Random Forest model after a 10 fold cross validation: %.3f (%.3f)\" % (mean(cv_scores), std(cv_scores)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "yhat = model.predict(X_test)\n",
    "\n",
    "# Computing Confusion matrix for the above algorithm\n",
    "\n",
    "cnf_matrix = confusion_matrix(y_test, yhat)\n",
    "\n",
    "np.set_printoptions(precision=2)\n",
    "\n",
    "# Plot non-normalized confusion matrix\n",
    "plt.figure()\n",
    "ri.plot_confusion_matrix(cnf_matrix, classes=[\"NON-DEFAULT\",\"DEFAULT\"])\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_df = ri.AppendResults(df=results_df, title=\"Full RF - Accepts Only\",\n",
    "AUC_scores=cv_scores, y_test=y_test, y_score=yhat)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Matching Characteristics ##"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Pre-processing ###"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "numerical_cols = ['loan_amnt', 'emp_length', 'dti']\n",
    "match_cols = ['loan_amnt', 'emp_length', 'dti','purpose','charged_off']\n",
    "excl_cols = ['charged_off','weight','score','rejected','accept_score']\n",
    "\n",
    "accept_train_df_reduced = accept_train_df[match_cols]\n",
    "accept_test_df_reduced = accept_test_df[match_cols]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "accept_train_df_reduced = pd.get_dummies(accept_train_df_reduced, prefix=[\"purp\"], columns=['purpose'], drop_first=False)\n",
    "accept_test_df_reduced = pd.get_dummies(accept_test_df_reduced, prefix=[\"purp\"], columns=['purpose'], drop_first=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "scaler_reduced = StandardScaler(copy=False)\n",
    "\n",
    "accept_train_df_reduced[numerical_cols] = accept_train_df_reduced[numerical_cols].fillna(accept_train_df_reduced[numerical_cols].mean())\n",
    "accept_test_df_reduced[numerical_cols] = accept_test_df_reduced[numerical_cols].fillna(accept_test_df_reduced[numerical_cols].mean())\n",
    "\n",
    "accept_train_df_reduced[numerical_cols] = scaler_reduced.fit_transform(accept_train_df_reduced[numerical_cols], accept_train_df_reduced['charged_off'])\n",
    "accept_test_df_reduced[numerical_cols] = scaler_reduced.transform(accept_test_df_reduced[numerical_cols])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "total, pos, rate = ri.splitCount(accept_train_df_reduced['charged_off'])\n",
    "print('Training Set:\\n    Total: {}\\n    Positive: {} ({:.2f}% of total)\\n'.format(\n",
    "    total, pos, rate))\n",
    "\n",
    "total, pos, rate = ri.splitCount(accept_test_df_reduced['charged_off'])\n",
    "print('Testing Set:\\n    Total: {}\\n    Positive: {} ({:.2f}% of total)\\n'.format(\n",
    "    total, pos, rate))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train_reduced = accept_train_df_reduced['charged_off']\n",
    "y_test_reduced = accept_test_df_reduced['charged_off']\n",
    "\n",
    "X_train_reduced = accept_train_df_reduced.drop('charged_off', axis=1)\n",
    "X_test_reduced = accept_test_df_reduced.drop('charged_off', axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Random Forest Model ###"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # # Hyperparameter tuning. Took over 11 hours to run.\n",
    "# # May be interesting to see if the optimised hyperparameters change with reduced characteristics\n",
    "\n",
    "# best_hyperparams, best_rf = ri.RandomForestTuning(BalancedRandomForestClassifier(), X_train_reduced, y_train_reduced, filename = \"./random_forest_reduced.joblib\", cv = CROSS_VALIDATION)\n",
    "\n",
    "# print(\"Best hyperparameters: \\n\", best_hyperparams )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_reduced = joblib.load(\"./random_forest_reduced.joblib\")\n",
    "print(model_reduced.get_params())\n",
    "model_reduced.fit(X_train_reduced, y_train_reduced)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cv_scores = cross_val_score(estimator= model_reduced, X=X_train_reduced, y=y_train_reduced, scoring='roc_auc', cv=CROSS_VALIDATION, n_jobs=-1)\n",
    "print(\"(Training Set) The cross-validation AUC for Reduced Random Forest model after a 10 fold cross validation: %.3f (%.3f)\" % (mean(cv_scores), std(cv_scores)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "yhat_reduced = model_reduced.predict(X_test_reduced)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_df = ri.AppendResults(df=results_df, title=\"Reduced RF - Accepts Only\",\n",
    "AUC_scores=cv_scores, y_test=y_test_reduced, y_score=yhat_reduced)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Reject Inference Techniques ##"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rejected = application_df.loc[application_df[\"rejected\"] == 1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Simple Assignment ###"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ri_df = pd.concat([accept_train_df, rejected], axis=0, ignore_index=True)\n",
    "ri_df.loc[ri_df['rejected'] == 1, 'charged_off'] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df_sa = ri_df[match_cols]\n",
    "train_df_sa = ri.RejectPreprocessing(df=train_df_sa, num_cols=numerical_cols, cat_cols=[\"purpose\"], prefix=[\"purp\"], scaler=scaler_reduced)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "total, pos, rate = ri.splitCount(train_df_sa['charged_off'])\n",
    "print('Training Set:\\n    Total: {}\\n    Positive: {} ({:.2f}% of total)\\n'.format(\n",
    "    total, pos, rate))\n",
    "\n",
    "total, pos, rate = ri.splitCount(accept_test_df_reduced['charged_off'])\n",
    "print('Testing Set:\\n    Total: {}\\n    Positive: {} ({:.2f}% of total)\\n'.format(\n",
    "    total, pos, rate))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train_sa = train_df_sa['charged_off']\n",
    "X_train_sa = train_df_sa.drop('charged_off', axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_ri = joblib.load(\"./random_forest_reduced.joblib\")\n",
    "print(model_ri.get_params())\n",
    "\n",
    "model_ri.fit(X_train_sa, y_train_sa)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cv_scores = cross_val_score(estimator= model_ri, X=X_train_sa, y=y_train_sa, scoring='roc_auc', cv=CROSS_VALIDATION, n_jobs=-1)\n",
    "print(\"(Training Set) The cross-validation AUC for optimised Random Forest model with SA RI after a 10 fold cross validation: %.3f (%.3f)\" % (mean(cv_scores), std(cv_scores)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "yhat_sa = model_ri.predict(X_test_reduced)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_df = ri.AppendResults(df=results_df, title=\"Reduced RF - Simple Assignment\",\n",
    "AUC_scores=cv_scores, y_test=y_test_reduced, y_score=yhat_sa)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Random Assignment ###"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "total, pos, reject_rate = ri.splitCount(accept_train_df['charged_off'])\n",
    "\n",
    "reject_rate = reject_rate / 100\n",
    "random_assign_rate = reject_rate * 3.5\n",
    "\n",
    "print(reject_rate)\n",
    "print(random_assign_rate)\n",
    "\n",
    "# Higher the fraction, bigger the sample\n",
    "split_BAD = rejected.sample(frac = random_assign_rate, random_state=RANDOM_STATE)\n",
    "split_GOOD = rejected.drop(split_BAD.index)\n",
    "\n",
    "split_BAD['charged_off'] = 1\n",
    "split_GOOD = split_GOOD.assign(charged_off= 0)\n",
    "\n",
    "ri_df = pd.concat([accept_train_df, split_BAD, split_GOOD], axis=0, ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df_ra = ri.RejectPreprocessing(df=ri_df[match_cols], num_cols=numerical_cols, cat_cols=[\"purpose\"], prefix=[\"purp\"], scaler=scaler_reduced)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "total, pos, rate = ri.splitCount(train_df_ra['charged_off'])\n",
    "print('Training Set:\\n    Total: {}\\n    Positive: {} ({:.2f}% of total)\\n'.format(\n",
    "    total, pos, rate))\n",
    "\n",
    "total, pos, rate = ri.splitCount(accept_test_df_reduced['charged_off'])\n",
    "print('Testing Set:\\n    Total: {}\\n    Positive: {} ({:.2f}% of total)\\n'.format(\n",
    "    total, pos, rate))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train_ra = train_df_ra['charged_off']\n",
    "X_train_ra = train_df_ra.drop('charged_off', axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_ri.fit(X_train_ra, y_train_ra)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cv_scores = cross_val_score(estimator= model_ri, X=X_train_ra, y=y_train_ra, scoring='roc_auc', cv=CROSS_VALIDATION, n_jobs=-1)\n",
    "print(\"(Training Set) The cross-validation AUC for optimised Random Forest model with RA RI after a 10 fold cross validation: %.3f (%.3f)\" % (mean(cv_scores), std(cv_scores)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "yhat_ra = model_reduced.predict(X_test_reduced)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_df = ri.AppendResults(df=results_df, title=\"Reduced RF - Random Assignment\",\n",
    "AUC_scores=cv_scores, y_test=y_test_reduced, y_score=yhat_ra)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Augmentation ###"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "aug_df = application_df.sort_values(by=\"score\", ascending = False)\n",
    "aug_df['accept_score'] = 1 - aug_df['score']\n",
    "intervals = np.array_split(aug_df, 10)\n",
    "\n",
    "for band in intervals:\n",
    "    accepts = band.loc[band[\"rejected\"] == 0, \"rejected\"].count()\n",
    "    rejects = band.loc[band[\"rejected\"] == 1, \"rejected\"].count()\n",
    "    weight = (rejects + accepts) / accepts\n",
    "    band.drop(band[band['rejected'] == 1].index, inplace=True)\n",
    "    band['weight'] = weight\n",
    "\n",
    "aug_df=pd.concat(intervals)\n",
    "aug_df.drop_duplicates(inplace=True)\n",
    "aug_df = accept_train_df.merge(aug_df, on=[\"dti\",\"emp_length\",\"loan_amnt\",\"purpose\"], how = 'inner')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df_aug_all = ri.RejectPreprocessing(df=aug_df, num_cols=all_cols, cat_cols=['home_ownership', 'verification_status', 'purpose', 'application_type'], prefix=[\"ho\",\"vs\",\"purp\",\"at\"], scaler=scaler_all)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "total, pos, rate= ri.splitCount(train_df_aug_all['charged_off'])\n",
    "print('Training Set:\\n    Total: {}\\n    Positive: {} ({:.2f}% of total)\\n'.format(\n",
    "    total, pos, rate))\n",
    "\n",
    "total, pos, rate = ri.splitCount(accept_test_df_all['charged_off'])\n",
    "print('Testing Set:\\n    Total: {}\\n    Positive: {} ({:.2f}% of total)\\n'.format(\n",
    "    total, pos, rate))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train_aug_all = train_df_aug_all['charged_off']\n",
    "X_train_aug_all = train_df_aug_all.drop(excl_cols, axis=1)\n",
    "train_weights_aug_all = train_df_aug_all['weight']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.fit(X_train_aug_all, y_train_aug_all, sample_weight=train_weights_aug_all)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cv_scores = cross_val_score(estimator= model, X=X_train_aug_all, y=y_train_aug_all, scoring='roc_auc', cv=CROSS_VALIDATION, n_jobs=-1)\n",
    "print(\"(Training Set) The cross-validation AUC for optimised Full Random Forest model with Augmentation after a 10 fold cross validation: %.3f (%.3f)\" % (mean(cv_scores), std(cv_scores)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "yhat_aug_all = model.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_df = ri.AppendResults(df=results_df, title=\"Full RF - Augmentation\",\n",
    "AUC_scores=cv_scores, y_test=y_test, y_score=yhat_aug_all)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Augmentation Reduced Columns ###"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "aug_cols = ['loan_amnt', 'emp_length', 'dti','purpose','charged_off',\"weight\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df_aug_reduced = ri.RejectPreprocessing(df=aug_df[aug_cols], num_cols=numerical_cols, cat_cols=['purpose'], prefix=[\"purp\"], scaler=scaler_reduced)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "total, pos, rate= ri.splitCount(train_df_aug_reduced['charged_off'])\n",
    "print('Training Set:\\n    Total: {}\\n    Positive: {} ({:.2f}% of total)\\n'.format(\n",
    "    total, pos, rate))\n",
    "\n",
    "total, pos, rate = ri.splitCount(accept_test_df_reduced['charged_off'])\n",
    "print('Testing Set:\\n    Total: {}\\n    Positive: {} ({:.2f}% of total)\\n'.format(\n",
    "    total, pos, rate))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train_aug_reduced = train_df_aug_reduced['charged_off']\n",
    "X_train_aug_reduced = train_df_aug_reduced.drop(['charged_off','weight'], axis=1)\n",
    "train_weights_aug_reduced = train_df_aug_reduced['weight']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_ri.fit(X_train_aug_reduced, y_train_aug_reduced, sample_weight=train_weights_aug_reduced)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cv_scores = cross_val_score(estimator= model_ri, X=X_train_aug_reduced, y=y_train_aug_reduced, scoring='roc_auc', cv=CROSS_VALIDATION, n_jobs=-1)\n",
    "print(\"(Training Set) The cross-validation AUC for optimised Full Random Forest model with Augmentation after a 10 fold cross validation: %.3f (%.3f)\" % (mean(cv_scores), std(cv_scores)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "yhat_aug_reduced = model_ri.predict(X_test_reduced)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_df = ri.AppendResults(df=results_df, title=\"Reduced RF - Augmentation\",\n",
    "AUC_scores=cv_scores, y_test=y_test, y_score=yhat_aug_reduced)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Parcelling ###"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "aug_df = application_df.sort_values(by=\"score\", ascending = False)\n",
    "\n",
    "parcel_df_reject = aug_df.loc[aug_df[\"rejected\"] == 1]\n",
    "parcel_df_accept = aug_df.loc[aug_df[\"rejected\"] == 0]\n",
    "\n",
    "parcel_df_accept = parcel_df_accept.drop_duplicates()\n",
    "\n",
    "accept_scores = accept_train_df.merge(parcel_df_accept, on=[\"dti\",\"emp_length\",\"loan_amnt\",\"purpose\"], how = 'left')\n",
    "\n",
    "# unsure of the number of intervals\n",
    "reject_intervals = np.array_split(parcel_df_reject, 10)\n",
    "accept_intervals = np.array_split(accept_scores, 10)\n",
    "\n",
    "counter = 0\n",
    "\n",
    "for reject_band in reject_intervals:\n",
    "\n",
    "    neg, pos = np.bincount(accept_intervals[counter]['charged_off'])\n",
    "    reject_rate = pos / (neg + pos)\n",
    "    assign_rate = reject_rate * 3.5\n",
    "\n",
    "    # Higher the fraction, bigger the sample\n",
    "    split_BAD = reject_band.sample(frac = assign_rate, random_state=RANDOM_STATE)\n",
    "    split_GOOD = reject_band.drop(split_BAD.index)\n",
    "\n",
    "    split_BAD['charged_off'] = 1\n",
    "    split_GOOD = split_GOOD.assign(charged_off= 0)\n",
    "    \n",
    "    accept_scores = pd.concat([accept_scores, split_BAD, split_GOOD], axis=0, ignore_index=True)\n",
    "    counter = counter + 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df_PARCEL_reduced = ri.RejectPreprocessing(df=accept_scores[match_cols], num_cols=numerical_cols, cat_cols=['purpose'], prefix=[\"purp\"], scaler=scaler_reduced)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "total, pos, rate = ri.splitCount(train_df_PARCEL_reduced['charged_off'])\n",
    "print('Training Set:\\n    Total: {}\\n    Positive: {} ({:.2f}% of total)\\n'.format(\n",
    "    total, pos, rate))\n",
    "\n",
    "total, pos, rate = ri.splitCount(accept_test_df_reduced['charged_off'])\n",
    "print('Testing Set:\\n    Total: {}\\n    Positive: {} ({:.2f}% of total)\\n'.format(\n",
    "    total, pos, rate))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train_PARCEL_reduced = train_df_PARCEL_reduced['charged_off']\n",
    "X_train_PARCEL_reduced = train_df_PARCEL_reduced.drop(['charged_off'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_ri.fit(X_train_PARCEL_reduced, y_train_PARCEL_reduced)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cv_scores = cross_val_score(estimator= model_ri, X=X_train_PARCEL_reduced, y=y_train_PARCEL_reduced, scoring='roc_auc', cv=CROSS_VALIDATION, n_jobs=-1)\n",
    "print(\"(Training Set) The cross-validation AUC for optimised Reduced Random Forest model with Parcelling after a 10 fold cross validation: %.3f (%.3f)\" % (mean(cv_scores), std(cv_scores)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "yhat_PARCEL_reduced = model_ri.predict(X_test_reduced)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_df = ri.AppendResults(df=results_df, title=\"Reduced RF - Parcelling\",\n",
    "AUC_scores=cv_scores, y_test=y_test_reduced, y_score=yhat_PARCEL_reduced)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tretsts = ri.AppendResults(df=results_df, title=\"Reduced RF - Parcelling\",\n",
    "AUC_scores=cv_scores, y_test=y_test_reduced, y_score=yhat_PARCEL_reduced)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Extrapolation ###"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rejects_extrapolation = rejected.drop(['rejected','score'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rejects_extrapolation = ri.RejectPreprocessing(df=rejects_extrapolation, num_cols=numerical_cols, cat_cols=['purpose'], prefix=[\"purp\"], scaler=scaler_reduced)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "reject_bad_prob = model_reduced.predict_proba(rejects_extrapolation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "total, pos, rate = ri.splitCount(accept_train_df_reduced['charged_off'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# No rejects scored below the rate because of the lack of variables\n",
    "# Suggestion for further work would be to change the rates\n",
    "\n",
    "print(\"Rate: \", rate)\n",
    "rate = 1 - (rate/100)\n",
    "\n",
    "rejects_extrapolation['bad_prob'] = reject_bad_prob[:,1]\n",
    "rejects_extrapolation['charged_off'] = np.where(rejects_extrapolation['bad_prob'] >= rate, 1, 0)\n",
    "rejects_extrapolation.drop('bad_prob', axis=1, inplace=True)\n",
    "\n",
    "train_df_EXTRAPOLATION_reduced = pd.concat([accept_train_df_reduced, rejects_extrapolation])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "total, pos, rate = ri.splitCount(train_df_EXTRAPOLATION_reduced['charged_off'])\n",
    "print('Training Set:\\n    Total: {}\\n    Positive: {} ({:.2f}% of total)\\n'.format(\n",
    "    total, pos, rate))\n",
    "\n",
    "total, pos, rate = ri.splitCount(accept_test_df_reduced['charged_off'])\n",
    "print('Testing Set:\\n    Total: {}\\n    Positive: {} ({:.2f}% of total)\\n'.format(\n",
    "    total, pos, rate))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train_EXTRAPOLATION_reduced = train_df_EXTRAPOLATION_reduced['charged_off']\n",
    "X_train_EXTRAPOLATION_reduced = train_df_EXTRAPOLATION_reduced.drop(['charged_off'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_ri.fit(X_train_EXTRAPOLATION_reduced, y_train_EXTRAPOLATION_reduced)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cv_scores = cross_val_score(estimator= model_ri, X=X_train_EXTRAPOLATION_reduced, y=y_train_EXTRAPOLATION_reduced, scoring='roc_auc', cv=CROSS_VALIDATION, n_jobs=-1)\n",
    "print(\"(Training Set) The cross-validation AUC for Reduced Random Forest - Exrapolated RI model after a 10 fold cross validation: %.3f (%.3f)\" % (mean(cv_scores), std(cv_scores)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "yhat_EXTRAPOLATION_reduced = model_ri.predict(X_test_reduced)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_df = ri.AppendResults(df=results_df, title=\"Reduced RF - Extrapolation\",\n",
    "AUC_scores=cv_scores, y_test=y_test_reduced, y_score=yhat_EXTRAPOLATION_reduced)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Fuzzy Augmentation ###"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rejected_bad = application_df.loc[application_df[\"rejected\"] == 1]\n",
    "rejected_good = application_df.loc[application_df[\"rejected\"] == 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rejected_good[\"score\"] = 1 - rejected_good[\"score\"]\n",
    "rejected_good[\"charged_off\"] = 0\n",
    "\n",
    "rejected_bad[\"charged_off\"] = 1\n",
    "\n",
    "fuzzy_aug = pd.concat([rejected_good, rejected_bad])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "accept_train_aug_df_reduced = accept_train_df[match_cols]\n",
    "accept_train_aug_df_reduced['score'] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_aug_df_reduced = pd.concat([accept_train_aug_df_reduced, fuzzy_aug])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_aug_df_reduced = ri.RejectPreprocessing(df=train_aug_df_reduced, num_cols=numerical_cols, cat_cols=['purpose'], prefix=[\"purp\"], scaler=scaler_reduced)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "total, pos, rate = ri.splitCount(train_aug_df_reduced['charged_off'])\n",
    "print('Training Set:\\n    Total: {}\\n    Positive: {} ({:.2f}% of total)\\n'.format(\n",
    "    total, pos, rate))\n",
    "\n",
    "total, pos, rate = ri.splitCount(y_test)\n",
    "print('Testing Set:\\n    Total: {}\\n    Positive: {} ({:.2f}% of total)\\n'.format(\n",
    "    total, pos, rate))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train_faug_reduced = train_aug_df_reduced['charged_off']\n",
    "X_train_faug_reduced = train_aug_df_reduced.drop(['charged_off',\"score\", 'rejected'], axis=1)\n",
    "train_weights_faug = train_aug_df_reduced['score']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_ri.fit(X_train_faug_reduced, y_train_faug_reduced, sample_weight = train_weights_faug)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cv_scores = cross_val_score(estimator= model_ri, X=X_train_faug_reduced, y=y_train_faug_reduced, scoring='roc_auc', cv=CROSS_VALIDATION, n_jobs=-1)\n",
    "print(\"(Training Set) The cross-validation AUC for Reduced Random Forest - Fuzzy Aug RI model after a 10 fold cross validation: %.3f (%.3f)\" % (mean(cv_scores), std(cv_scores)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "yhat_faug_reduced = model_ri.predict(X_test_reduced)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_df = ri.AppendResults(df=results_df, title=\"Reduced RF - Fuzzy Augmentation\",\n",
    "AUC_scores=cv_scores, y_test=y_test, y_score=yhat_faug_reduced)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Bayesian Network ###"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bn_df = accept_train_aug_df_reduced\n",
    "emp_length_mean = bn_df['emp_length'].mean()\n",
    "bn_df['emp_length'].fillna(emp_length_mean, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bn_df['emp_length_cat'], emp_bins = pd.qcut(bn_df['emp_length'], q=3, retbins=True)\n",
    "bn_df['dti_cat'], dti_bins = pd.qcut(bn_df['dti'], q=4, retbins=True)\n",
    "bn_df['loan_amnt_cat'], loan_bins = pd.qcut(bn_df['loan_amnt'], q=4, retbins=True)\n",
    "\n",
    "bn_df['emp_length_cat'] = bn_df['emp_length_cat'].astype(str)\n",
    "bn_df['dti_cat'] = bn_df['dti_cat'].astype(str)\n",
    "bn_df['loan_amnt_cat'] = bn_df['loan_amnt_cat'].astype(str)\n",
    "\n",
    "emp_length_cat_lb = str(emp_bins[-1])\n",
    "dti_cat_lb = str(dti_bins[-1])\n",
    "loan_amnt_cat_lb = str(loan_bins[-1])\n",
    "\n",
    "emp_length_open_cat = str(emp_bins[-2]) + \" <\"\n",
    "dti_open_cat = str(dti_bins[-2]) + \" <\"\n",
    "loan_amnt_open_cat = str(loan_bins[-2]) + \" <\"\n",
    "\n",
    "bn_df.loc[bn_df[\"emp_length_cat\"].str.contains(emp_length_cat_lb), \"emp_length_cat\"] = emp_length_open_cat\n",
    "bn_df.loc[bn_df[\"dti_cat\"].str.contains(dti_cat_lb), \"dti_cat\"] = dti_open_cat\n",
    "bn_df.loc[bn_df[\"loan_amnt_cat\"].str.contains(loan_amnt_cat_lb), \"loan_amnt_cat\"] = loan_amnt_open_cat\n",
    "\n",
    "bn_df = bn_df.drop(['loan_amnt', 'emp_length', 'dti','score'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bn_rejects = rejected\n",
    "bn_rejects['emp_length'].fillna(emp_length_mean, inplace = True)\n",
    "bn_rejects = bn_rejects.drop(['rejected', 'score'], axis=1)\n",
    "\n",
    "# Need to bin the reject continous variables\n",
    "bn_rejects['emp_length_cat'] = pd.cut(bn_rejects[\"emp_length\"], bins=emp_bins, include_lowest=True).astype(str)\n",
    "bn_rejects['dti_cat'] = pd.cut(bn_rejects[\"dti\"], bins=dti_bins, include_lowest=True).astype(str)\n",
    "bn_rejects['loan_amnt_cat'] = pd.cut(bn_rejects[\"loan_amnt\"], bins=loan_bins, include_lowest=True).astype(str)\n",
    "\n",
    "# Filling values that are in the rejcted dataset outside the threshold of the application dataset bins\n",
    "bn_rejects['dti_cat'].fillna(dti_open_cat, inplace = True)\n",
    "bn_rejects['loan_amnt_cat'].fillna(loan_amnt_open_cat, inplace = True)\n",
    "\n",
    "bn_rejects.loc[bn_rejects[\"emp_length_cat\"].str.contains(emp_length_cat_lb), \"emp_length_cat\"] = emp_length_open_cat\n",
    "bn_rejects.loc[bn_rejects[\"dti_cat\"].str.contains(dti_cat_lb), \"dti_cat\"] = dti_open_cat\n",
    "bn_rejects.loc[bn_rejects[\"loan_amnt_cat\"].str.contains(loan_amnt_cat_lb), \"loan_amnt_cat\"] = loan_amnt_open_cat\n",
    "\n",
    "bn_rejects = bn_rejects.drop(['loan_amnt', 'emp_length', 'dti'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(bn_rejects.columns)\n",
    "print(list(set(bn_rejects['purpose']) - set(bn_df['purpose'])))\n",
    "print(list(set(bn_df['purpose']) - set(bn_rejects['purpose'])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Network structure optimisation\n",
    "\n",
    "scoring_method = BicScore(data=bn_df)\n",
    "# est = HillClimbSearch(data=df)\n",
    "est = ExhaustiveSearch(data=bn_df, scoring_method=scoring_method)\n",
    "estimated_model = est.estimate()\n",
    "\n",
    "print(estimated_model.edges())\n",
    "\n",
    "print(\"\\nAll DAGs by score:\")\n",
    "for score, dag in reversed(est.all_scores()):\n",
    "    print(score, dag.edges())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bn_model = BayesianNetwork(estimated_model)\n",
    "bn_model.fit(bn_df, estimator=BayesianEstimator, prior_type=\"BDeu\") # Bayesian Dirichlet equivalent uniform prior\n",
    "for cpd in bn_model.get_cpds():\n",
    "    print(cpd)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "joblib.dump(bn_model, \"./BN model.joblib\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "chunks = np.array_split(bn_rejects, 400)\n",
    "\n",
    "for i in chunks:\n",
    "    bn_yhat = bn_model.predict_probability(i)\n",
    "    i['default_prob'] = bn_yhat['charged_off_1']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bn_yhat_df=pd.concat(chunks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save to CSV file\n",
    "bn_yhat_df.to_csv('BN Results.csv', mode='w+')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "total, pos, rate = ri.splitCount(bn_df['charged_off'])\n",
    "reject_rate = rate / 100\n",
    "\n",
    "bn_yhat_df['charged_off'] = np.where(bn_yhat_df['default_prob'] >= reject_rate, 1, 0)\n",
    "\n",
    "rejected['charged_off'] = bn_yhat_df['charged_off']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_bn_reduced = pd.concat([rejected, accept_train_aug_df_reduced])\n",
    "train_bn_reduced = train_bn_reduced[match_cols]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_bn_reduced = ri.RejectPreprocessing(df=train_bn_reduced, num_cols=numerical_cols, cat_cols=['purpose'], prefix=[\"purp\"], scaler=scaler_reduced)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "total, pos, rate = ri.splitCount(train_bn_reduced['charged_off'])\n",
    "print('Training Set:\\n    Total: {}\\n    Positive: {} ({:.2f}% of total)\\n'.format(\n",
    "    total, pos, rate))\n",
    "\n",
    "total, pos, rate = ri.splitCount(accept_test_df_reduced['charged_off'])\n",
    "print('Testing Set:\\n    Total: {}\\n    Positive: {} ({:.2f}% of total)\\n'.format(\n",
    "    total, pos, rate))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train_bn_reduced = train_bn_reduced['charged_off']\n",
    "X_train_bn_reduced = train_bn_reduced.drop(['charged_off'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_ri.fit(X_train_bn_reduced, y_train_bn_reduced)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cv_scores = cross_val_score(estimator= model_ri, X=X_train_bn_reduced, y=y_train_bn_reduced, scoring='roc_auc', cv=CROSS_VALIDATION, n_jobs=-1)\n",
    "print(\"(Training Set) The cross-validation AUC for Reduced Random Forest - Bayesian Network RI model after a 10 fold cross validation: %.3f (%.3f)\" % (mean(cv_scores), std(cv_scores)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "yhat_bn_reduced = model_ri.predict(X_test_reduced)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_df = ri.AppendResults(df=results_df, title=\"Reduced RF - Bayesian Network\",\n",
    "AUC_scores=cv_scores, y_test=y_test, y_score=yhat_bn_reduced)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save to CSV file\n",
    "results_df.to_csv('Results.csv', mode='w+')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Visuals ##"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This may not the best way to view each estimator as it is small\n",
    "\n",
    "fig, axes = plt.subplots(nrows = 1,ncols = 5,figsize = (10,2), dpi=3000)\n",
    "for index in range(0, 5):\n",
    "    tree.plot_tree(model_reduced.estimators_[index],\n",
    "                   feature_names = X_train_reduced.columns,\n",
    "                   filled = True,\n",
    "                   ax = axes[index]);\n",
    "    \n",
    "    axes[index].set_title('Estimator: ' + str(index), fontsize = 11)\n",
    "fig.savefig('rf_5trees_reduced.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, axes = plt.subplots(nrows = 1,ncols = 1,figsize = (5,5), dpi=2000)\n",
    "tree.plot_tree(model_reduced.estimators_[25],\n",
    "               feature_names = X_train_reduced.columns,\n",
    "               filled = True, proportion = False,precision = 2, rounded = True, fontsize=3);\n",
    "\n",
    "fig.savefig('rf_accept_reduced_single_tree.png')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.8.8 ('base')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  },
  "vscode": {
   "interpreter": {
    "hash": "653ef3dff1567b17fc742ba82e2411e657a18e82067de2bcca7734cac483dd3c"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}