3.1_data_sampling.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# This notebook is used to\n",
    "# 1. Refine the classifiers to mitigate data imbalance\n",
    "# 2. Create ensemble classifier\n",
    "# 3. Perform feature ablation\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "import copy\n",
    "from joblib import dump, load\n",
    "\n",
    "plt.style.use('bmh')\n",
    "mpl.rcParams.update({\n",
    "    \"grid.linestyle\" : \"dashed\",\n",
    "    \"axes.facecolor\" : \"white\",\n",
    "    \"axes.spines.top\" : False,\n",
    "    \"axes.spines.right\" : False,\n",
    "    \"legend.frameon\" : False,\n",
    "    \"figure.figsize\" : (8, 5),\n",
    "    \"figure.dpi\" : 300,\n",
    "})\n",
    "\n",
    "# suppress sklearn deprecated warnings\n",
    "import warnings\n",
    "def warn(*args, **kwargs): pass\n",
    "warnings.warn = warn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "models/32features/mean/\n",
      "data/32features/mean/\n"
     ]
    }
   ],
   "source": [
    "# TODO make test for dataset with creatina column\n",
    "# Dataset without thyroid = 18 features (including survive7y)\n",
    "# Dataset with thyroid = 27 features (including survive7y)\n",
    "# With columns that have missing values, 23 and 32\n",
    "# Default 18\n",
    "n_features = 32\n",
    "extra_path = n_features != 27 and n_features != 18\n",
    "dropped_na_key = \"dropped_na/\"\n",
    "mean_key = \"mean/\"\n",
    "path = f\"data/{n_features}features/{mean_key if extra_path else '' }\"\n",
    "path_models = f\"models/{n_features}features/{mean_key if extra_path else '' }\"\n",
    "path_images = f\"figures/{n_features}features/{mean_key if extra_path else '' }\"\n",
    "output_models = f\"models_output/{n_features}features/{mean_key if extra_path else '' }\"\n",
    "print(path_models)\n",
    "print(path)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Sampling\n",
    "Oversample and undersample methods to mitigate data imbalance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6447\n"
     ]
    }
   ],
   "source": [
    "# Load Data for sampling\n",
    "df_train = pd.read_csv(f\"{path}train.csv\", index_col=0)\n",
    "df_valid = pd.read_csv(f\"{path}valid.csv\", index_col=0)\n",
    "df_test = pd.read_csv(f\"{path}test.csv\", index_col=0)\n",
    "print(len(df_train)+len(df_valid)+len(df_test))\n",
    "feat_names = list(df_train.columns)\n",
    "train, valid, test = df_train.to_numpy(), df_valid.to_numpy(), df_test.to_numpy()\n",
    "X_train, y_train = train[:, :-1], train[:, -1]\n",
    "X_valid, y_valid = valid[:, :-1], valid[:, -1]\n",
    "X_test, y_test = test[:, :-1], test[:, -1]\n",
    "\n",
    "from utils import get_preprocess_std_num\n",
    "preprocess_std = get_preprocess_std_num(feat_names)\n",
    "\n",
    "\n",
    "# Preprocessed ready-to-use train and valid set\n",
    "process_tmp = preprocess_std.fit(X_train)\n",
    "X_train_std = process_tmp.transform(X_train)\n",
    "X_valid_std = process_tmp.transform(X_valid)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------------------------------------------\n",
      "Oversample          Score               Random 0.2 Name lr\n",
      "-------------------------------------------------\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.665\n",
      "bordersmote         0.632\n",
      "svmsmote            0.675\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.665\n",
      "bordersmote         0.654\n",
      "svmsmote            0.657\n",
      "-------------------------------------------------\n",
      "Oversample          Score               Random 0.25 Name lr\n",
      "-------------------------------------------------\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.646\n",
      "bordersmote         0.686\n",
      "svmsmote            0.681\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.670\n",
      "bordersmote         0.658\n",
      "svmsmote            0.682\n",
      "-------------------------------------------------\n",
      "Oversample          Score               Random 0.3 Name lr\n",
      "-------------------------------------------------\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.662\n",
      "bordersmote         0.662\n",
      "svmsmote            0.671\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.664\n",
      "bordersmote         0.632\n",
      "svmsmote            0.663\n",
      "-------------------------------------------------\n",
      "Oversample          Score               Random 0.35 Name lr\n",
      "-------------------------------------------------\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.683\n",
      "bordersmote         0.679\n",
      "svmsmote            0.664\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.629\n",
      "bordersmote         0.630\n",
      "svmsmote            0.668\n",
      "-------------------------------------------------\n",
      "Oversample          Score               Random 0.4 Name lr\n",
      "-------------------------------------------------\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.672\n",
      "bordersmote         0.647\n",
      "svmsmote            0.676\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.644\n",
      "bordersmote         0.639\n",
      "svmsmote            0.646\n",
      "-------------------------------------------------\n",
      "Oversample          Score               Random 0.45 Name lr\n",
      "-------------------------------------------------\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.669\n",
      "bordersmote         0.655\n",
      "svmsmote            0.684\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.666\n",
      "bordersmote         0.638\n",
      "svmsmote            0.654\n",
      "-------------------------------------------------\n",
      "Oversample          Score               Random 0.5 Name lr\n",
      "-------------------------------------------------\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.675\n",
      "bordersmote         0.662\n",
      "svmsmote            0.649\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.674\n",
      "bordersmote         0.661\n",
      "svmsmote            0.693\n",
      "\n",
      "\n",
      "Name model:      \tlr\n",
      "Sampling method: undersampling + oversampling\n",
      "Best rand_ratio: \t0.5\n",
      "Best score:      \t0.6927493245274986\n",
      "Dataset size:    \t2020, Counter({0.0: 1010, 1.0: 1010})\n",
      "Combination:     \t_random_svmsmote_lr\n",
      "\n",
      "\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "         0.0      0.785     0.716     0.749      1010\n",
      "         1.0      0.739     0.804     0.770      1010\n",
      "\n",
      "    accuracy                          0.760      2020\n",
      "   macro avg      0.762     0.760     0.759      2020\n",
      "weighted avg      0.762     0.760     0.759      2020\n",
      "\n",
      "auc macro 0.853\n",
      "confusion matrix\n",
      "[[723 287]\n",
      " [198 812]]\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "         0.0      0.372     0.664     0.477       149\n",
      "         1.0      0.948     0.844     0.893      1072\n",
      "\n",
      "    accuracy                          0.822      1221\n",
      "   macro avg      0.660     0.754     0.685      1221\n",
      "weighted avg      0.877     0.822     0.842      1221\n",
      "\n",
      "auc macro 0.840\n",
      "confusion matrix\n",
      "[[ 99  50]\n",
      " [167 905]]\n",
      "-------------------------------------------------\n",
      "Oversample          Score               Random 0.2 Name svc\n",
      "-------------------------------------------------\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.598\n",
      "bordersmote         0.560\n",
      "svmsmote            0.381\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.510\n",
      "bordersmote         0.621\n",
      "svmsmote            0.633\n",
      "-------------------------------------------------\n",
      "Oversample          Score               Random 0.25 Name svc\n",
      "-------------------------------------------------\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.618\n",
      "bordersmote         0.600\n",
      "svmsmote            0.476\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.637\n",
      "bordersmote         0.660\n",
      "svmsmote            0.645\n",
      "-------------------------------------------------\n",
      "Oversample          Score               Random 0.3 Name svc\n",
      "-------------------------------------------------\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.540\n",
      "bordersmote         0.430\n",
      "svmsmote            0.579\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.514\n",
      "bordersmote         0.628\n",
      "svmsmote            0.641\n",
      "-------------------------------------------------\n",
      "Oversample          Score               Random 0.35 Name svc\n",
      "-------------------------------------------------\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.507\n",
      "bordersmote         0.588\n",
      "svmsmote            0.443\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.656\n",
      "bordersmote         0.628\n",
      "svmsmote            0.626\n",
      "-------------------------------------------------\n",
      "Oversample          Score               Random 0.4 Name svc\n",
      "-------------------------------------------------\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.613\n",
      "bordersmote         0.509\n",
      "svmsmote            0.466\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.660\n",
      "bordersmote         0.633\n",
      "svmsmote            0.646\n",
      "-------------------------------------------------\n",
      "Oversample          Score               Random 0.45 Name svc\n",
      "-------------------------------------------------\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.449\n",
      "bordersmote         0.517\n",
      "svmsmote            0.558\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.657\n",
      "bordersmote         0.551\n",
      "svmsmote            0.627\n",
      "-------------------------------------------------\n",
      "Oversample          Score               Random 0.5 Name svc\n",
      "-------------------------------------------------\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.574\n",
      "bordersmote         0.594\n",
      "svmsmote            0.562\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.657\n",
      "bordersmote         0.623\n",
      "svmsmote            0.652\n",
      "\n",
      "\n",
      "Name model:      \tsvc\n",
      "Sampling method: undersampling + oversampling\n",
      "Best rand_ratio: \t0.35\n",
      "Best score:      \t0.6865729680272454\n",
      "Dataset size:    \t2884, Counter({0.0: 1442, 1.0: 1442})\n",
      "Combination:     \t_random_svmsmote_svc\n",
      "\n",
      "\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "         0.0      0.798     0.621     0.698      1442\n",
      "         1.0      0.690     0.843     0.758      1442\n",
      "\n",
      "    accuracy                          0.732      2884\n",
      "   macro avg      0.744     0.732     0.728      2884\n",
      "weighted avg      0.744     0.732     0.728      2884\n",
      "\n",
      "auc macro 0.836\n",
      "confusion matrix\n",
      "[[ 895  547]\n",
      " [ 227 1215]]\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "         0.0      0.386     0.604     0.471       149\n",
      "         1.0      0.940     0.867     0.902      1072\n",
      "\n",
      "    accuracy                          0.835      1221\n",
      "   macro avg      0.663     0.735     0.687      1221\n",
      "weighted avg      0.873     0.835     0.849      1221\n",
      "\n",
      "auc macro 0.810\n",
      "confusion matrix\n",
      "[[ 90  59]\n",
      " [143 929]]\n",
      "-------------------------------------------------\n",
      "Oversample          Score               Random 0.2 Name knn\n",
      "-------------------------------------------------\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.553\n",
      "bordersmote         0.584\n",
      "svmsmote            0.600\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.531\n",
      "bordersmote         0.549\n",
      "svmsmote            0.558\n",
      "-------------------------------------------------\n",
      "Oversample          Score               Random 0.25 Name knn\n",
      "-------------------------------------------------\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.550\n",
      "bordersmote         0.576\n",
      "svmsmote            0.594\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.537\n",
      "bordersmote         0.572\n",
      "svmsmote            0.576\n",
      "-------------------------------------------------\n",
      "Oversample          Score               Random 0.3 Name knn\n",
      "-------------------------------------------------\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.547\n",
      "bordersmote         0.565\n",
      "svmsmote            0.579\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.550\n",
      "bordersmote         0.541\n",
      "svmsmote            0.583\n",
      "-------------------------------------------------\n",
      "Oversample          Score               Random 0.35 Name knn\n",
      "-------------------------------------------------\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.557\n",
      "bordersmote         0.581\n",
      "svmsmote            0.582\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.537\n",
      "bordersmote         0.538\n",
      "svmsmote            0.573\n",
      "-------------------------------------------------\n",
      "Oversample          Score               Random 0.4 Name knn\n",
      "-------------------------------------------------\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.561\n",
      "bordersmote         0.582\n",
      "svmsmote            0.586\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.522\n",
      "bordersmote         0.522\n",
      "svmsmote            0.573\n",
      "-------------------------------------------------\n",
      "Oversample          Score               Random 0.45 Name knn\n",
      "-------------------------------------------------\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.556\n",
      "bordersmote         0.566\n",
      "svmsmote            0.593\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.533\n",
      "bordersmote         0.530\n",
      "svmsmote            0.570\n",
      "-------------------------------------------------\n",
      "Oversample          Score               Random 0.5 Name knn\n",
      "-------------------------------------------------\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.549\n",
      "bordersmote         0.569\n",
      "svmsmote            0.581\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.550\n",
      "bordersmote         0.520\n",
      "svmsmote            0.565\n",
      "\n",
      "\n",
      "Name model:      \tknn\n",
      "Sampling method: only oversampling\n",
      "Best rand_ratio: \t0.45\n",
      "Best score:      \t0.6002350677068247\n",
      "Dataset size:    \t6988, Counter({1.0: 3494, 0.0: 3494})\n",
      "Combination:     \t_random_svmsmote_knn\n",
      "\n",
      "\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "         0.0      0.859     0.986     0.918      3494\n",
      "         1.0      0.983     0.838     0.905      3494\n",
      "\n",
      "    accuracy                          0.912      6988\n",
      "   macro avg      0.921     0.912     0.912      6988\n",
      "weighted avg      0.921     0.912     0.912      6988\n",
      "\n",
      "auc macro 0.992\n",
      "confusion matrix\n",
      "[[3444   50]\n",
      " [ 565 2929]]\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "         0.0      0.260     0.530     0.349       149\n",
      "         1.0      0.924     0.790     0.852      1072\n",
      "\n",
      "    accuracy                          0.758      1221\n",
      "   macro avg      0.592     0.660     0.600      1221\n",
      "weighted avg      0.843     0.758     0.790      1221\n",
      "\n",
      "auc macro 0.695\n",
      "confusion matrix\n",
      "[[ 79  70]\n",
      " [225 847]]\n",
      "-------------------------------------------------\n",
      "Oversample          Score               Random 0.2 Name rf\n",
      "-------------------------------------------------\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.628\n",
      "bordersmote         0.632\n",
      "svmsmote            0.640\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.652\n",
      "bordersmote         0.662\n",
      "svmsmote            0.670\n",
      "-------------------------------------------------\n",
      "Oversample          Score               Random 0.25 Name rf\n",
      "-------------------------------------------------\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.624\n",
      "bordersmote         0.626\n",
      "svmsmote            0.645\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.677\n",
      "bordersmote         0.677\n",
      "svmsmote            0.690\n",
      "-------------------------------------------------\n",
      "Oversample          Score               Random 0.3 Name rf\n",
      "-------------------------------------------------\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.627\n",
      "bordersmote         0.630\n",
      "svmsmote            0.643\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.698\n",
      "bordersmote         0.706\n",
      "svmsmote            0.693\n",
      "-------------------------------------------------\n",
      "Oversample          Score               Random 0.35 Name rf\n",
      "-------------------------------------------------\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.628\n",
      "bordersmote         0.627\n",
      "svmsmote            0.635\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.697\n",
      "bordersmote         0.701\n",
      "svmsmote            0.699\n",
      "-------------------------------------------------\n",
      "Oversample          Score               Random 0.4 Name rf\n",
      "-------------------------------------------------\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.618\n",
      "bordersmote         0.633\n",
      "svmsmote            0.650\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.711\n",
      "bordersmote         0.702\n",
      "svmsmote            0.688\n",
      "-------------------------------------------------\n",
      "Oversample          Score               Random 0.45 Name rf\n",
      "-------------------------------------------------\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.632\n",
      "bordersmote         0.625\n",
      "svmsmote            0.636\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.699\n",
      "bordersmote         0.689\n",
      "svmsmote            0.693\n",
      "-------------------------------------------------\n",
      "Oversample          Score               Random 0.5 Name rf\n",
      "-------------------------------------------------\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.620\n",
      "bordersmote         0.624\n",
      "svmsmote            0.637\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.692\n",
      "bordersmote         0.692\n",
      "svmsmote            0.702\n",
      "\n",
      "\n",
      "Name model:      \trf\n",
      "Sampling method: undersampling + oversampling\n",
      "Best rand_ratio: \t0.4\n",
      "Best score:      \t0.7112227975890704\n",
      "Dataset size:    \t2524, Counter({0.0: 1262, 1.0: 1262})\n",
      "Combination:     \t_random_smote_rf\n",
      "\n",
      "\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "         0.0      0.977     0.941     0.959      1262\n",
      "         1.0      0.943     0.978     0.960      1262\n",
      "\n",
      "    accuracy                          0.960      2524\n",
      "   macro avg      0.960     0.960     0.960      2524\n",
      "weighted avg      0.960     0.960     0.960      2524\n",
      "\n",
      "auc macro 0.994\n",
      "confusion matrix\n",
      "[[1188   74]\n",
      " [  28 1234]]\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "         0.0      0.449     0.537     0.489       149\n",
      "         1.0      0.934     0.909     0.921      1072\n",
      "\n",
      "    accuracy                          0.863      1221\n",
      "   macro avg      0.692     0.723     0.705      1221\n",
      "weighted avg      0.875     0.863     0.868      1221\n",
      "\n",
      "auc macro 0.848\n",
      "confusion matrix\n",
      "[[ 80  69]\n",
      " [ 98 974]]\n",
      "-------------------------------------------------\n",
      "Oversample          Score               Random 0.2 Name adaboost\n",
      "-------------------------------------------------\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.645\n",
      "bordersmote         0.662\n",
      "svmsmote            0.651\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.651\n",
      "bordersmote         0.678\n",
      "svmsmote            0.661\n",
      "-------------------------------------------------\n",
      "Oversample          Score               Random 0.25 Name adaboost\n",
      "-------------------------------------------------\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.669\n",
      "bordersmote         0.672\n",
      "svmsmote            0.670\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.660\n",
      "bordersmote         0.676\n",
      "svmsmote            0.660\n",
      "-------------------------------------------------\n",
      "Oversample          Score               Random 0.3 Name adaboost\n",
      "-------------------------------------------------\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.658\n",
      "bordersmote         0.647\n",
      "svmsmote            0.665\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.690\n",
      "bordersmote         0.668\n",
      "svmsmote            0.652\n",
      "-------------------------------------------------\n",
      "Oversample          Score               Random 0.35 Name adaboost\n",
      "-------------------------------------------------\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.655\n",
      "bordersmote         0.657\n",
      "svmsmote            0.632\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.676\n",
      "bordersmote         0.659\n",
      "svmsmote            0.676\n",
      "-------------------------------------------------\n",
      "Oversample          Score               Random 0.4 Name adaboost\n",
      "-------------------------------------------------\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.661\n",
      "bordersmote         0.683\n",
      "svmsmote            0.652\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.690\n",
      "bordersmote         0.659\n",
      "svmsmote            0.668\n",
      "-------------------------------------------------\n",
      "Oversample          Score               Random 0.45 Name adaboost\n",
      "-------------------------------------------------\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.646\n",
      "bordersmote         0.647\n",
      "svmsmote            0.656\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.640\n",
      "bordersmote         0.678\n",
      "svmsmote            0.657\n",
      "-------------------------------------------------\n",
      "Oversample          Score               Random 0.5 Name adaboost\n",
      "-------------------------------------------------\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.691\n",
      "bordersmote         0.659\n",
      "svmsmote            0.665\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.664\n",
      "bordersmote         0.673\n",
      "svmsmote            0.684\n",
      "\n",
      "\n",
      "Name model:      \tadaboost\n",
      "Sampling method: undersampling + oversampling\n",
      "Best rand_ratio: \t0.3\n",
      "Best score:      \t0.6998361103523627\n",
      "Dataset size:    \t3366, Counter({0.0: 1683, 1.0: 1683})\n",
      "Combination:     \t_random_smote_adaboost\n",
      "\n",
      "\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "         0.0      0.873     0.835     0.854      1683\n",
      "         1.0      0.842     0.878     0.860      1683\n",
      "\n",
      "    accuracy                          0.857      3366\n",
      "   macro avg      0.857     0.857     0.857      3366\n",
      "weighted avg      0.857     0.857     0.857      3366\n",
      "\n",
      "auc macro 0.938\n",
      "confusion matrix\n",
      "[[1406  277]\n",
      " [ 205 1478]]\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "         0.0      0.423     0.570     0.486       149\n",
      "         1.0      0.937     0.892     0.914      1072\n",
      "\n",
      "    accuracy                          0.853      1221\n",
      "   macro avg      0.680     0.731     0.700      1221\n",
      "weighted avg      0.874     0.853     0.862      1221\n",
      "\n",
      "auc macro 0.826\n",
      "confusion matrix\n",
      "[[ 85  64]\n",
      " [116 956]]\n",
      "-------------------------------------------------\n",
      "Oversample          Score               Random 0.2 Name nn\n",
      "-------------------------------------------------\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.647\n",
      "bordersmote         0.657\n",
      "svmsmote            0.666\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.660\n",
      "bordersmote         0.666\n",
      "svmsmote            0.660\n",
      "-------------------------------------------------\n",
      "Oversample          Score               Random 0.25 Name nn\n",
      "-------------------------------------------------\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.662\n",
      "bordersmote         0.653\n",
      "svmsmote            0.672\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.659\n",
      "bordersmote         0.661\n",
      "svmsmote            0.661\n",
      "-------------------------------------------------\n",
      "Oversample          Score               Random 0.3 Name nn\n",
      "-------------------------------------------------\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.646\n",
      "bordersmote         0.658\n",
      "svmsmote            0.681\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.656\n",
      "bordersmote         0.658\n",
      "svmsmote            0.658\n",
      "-------------------------------------------------\n",
      "Oversample          Score               Random 0.35 Name nn\n",
      "-------------------------------------------------\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.658\n",
      "bordersmote         0.649\n",
      "svmsmote            0.664\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.650\n",
      "bordersmote         0.654\n",
      "svmsmote            0.675\n",
      "-------------------------------------------------\n",
      "Oversample          Score               Random 0.4 Name nn\n",
      "-------------------------------------------------\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.664\n",
      "bordersmote         0.668\n",
      "svmsmote            0.655\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.665\n",
      "bordersmote         0.647\n",
      "svmsmote            0.654\n",
      "-------------------------------------------------\n",
      "Oversample          Score               Random 0.45 Name nn\n",
      "-------------------------------------------------\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.652\n",
      "bordersmote         0.659\n",
      "svmsmote            0.667\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.642\n",
      "bordersmote         0.637\n",
      "svmsmote            0.655\n",
      "-------------------------------------------------\n",
      "Oversample          Score               Random 0.5 Name nn\n",
      "-------------------------------------------------\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.654\n",
      "bordersmote         0.656\n",
      "svmsmote            0.683\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.639\n",
      "bordersmote         0.668\n",
      "svmsmote            0.652\n",
      "\n",
      "\n",
      "Name model:      \tnn\n",
      "Sampling method: undersampling + oversampling\n",
      "Best rand_ratio: \t0.25\n",
      "Best score:      \t0.6905421305081808\n",
      "Dataset size:    \t4040, Counter({0.0: 2020, 1.0: 2020})\n",
      "Combination:     \t_random_svmsmote_nn\n",
      "\n",
      "\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "         0.0      0.856     0.861     0.858      2020\n",
      "         1.0      0.860     0.855     0.857      2020\n",
      "\n",
      "    accuracy                          0.858      4040\n",
      "   macro avg      0.858     0.858     0.858      4040\n",
      "weighted avg      0.858     0.858     0.858      4040\n",
      "\n",
      "auc macro 0.930\n",
      "confusion matrix\n",
      "[[1739  281]\n",
      " [ 293 1727]]\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "         0.0      0.401     0.624     0.488       149\n",
      "         1.0      0.943     0.870     0.905      1072\n",
      "\n",
      "    accuracy                          0.840      1221\n",
      "   macro avg      0.672     0.747     0.697      1221\n",
      "weighted avg      0.877     0.840     0.854      1221\n",
      "\n",
      "auc macro 0.847\n",
      "confusion matrix\n",
      "[[ 93  56]\n",
      " [139 933]]\n",
      "-------------------------------------------------\n",
      "Oversample          Score               Random 0.2 Name gb\n",
      "-------------------------------------------------\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.649\n",
      "bordersmote         0.662\n",
      "svmsmote            0.688\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.678\n",
      "bordersmote         0.680\n",
      "svmsmote            0.681\n",
      "-------------------------------------------------\n",
      "Oversample          Score               Random 0.25 Name gb\n",
      "-------------------------------------------------\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.655\n",
      "bordersmote         0.656\n",
      "svmsmote            0.679\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.692\n",
      "bordersmote         0.692\n",
      "svmsmote            0.692\n",
      "-------------------------------------------------\n",
      "Oversample          Score               Random 0.3 Name gb\n",
      "-------------------------------------------------\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.647\n",
      "bordersmote         0.658\n",
      "svmsmote            0.672\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.695\n",
      "bordersmote         0.694\n",
      "svmsmote            0.683\n",
      "-------------------------------------------------\n",
      "Oversample          Score               Random 0.35 Name gb\n",
      "-------------------------------------------------\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.653\n",
      "bordersmote         0.664\n",
      "svmsmote            0.677\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.700\n",
      "bordersmote         0.692\n",
      "svmsmote            0.690\n",
      "-------------------------------------------------\n",
      "Oversample          Score               Random 0.4 Name gb\n",
      "-------------------------------------------------\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.655\n",
      "bordersmote         0.653\n",
      "svmsmote            0.679\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.695\n",
      "bordersmote         0.696\n",
      "svmsmote            0.694\n",
      "-------------------------------------------------\n",
      "Oversample          Score               Random 0.45 Name gb\n",
      "-------------------------------------------------\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.646\n",
      "bordersmote         0.664\n",
      "svmsmote            0.682\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.679\n",
      "bordersmote         0.678\n",
      "svmsmote            0.691\n",
      "-------------------------------------------------\n",
      "Oversample          Score               Random 0.5 Name gb\n",
      "-------------------------------------------------\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.652\n",
      "bordersmote         0.655\n",
      "svmsmote            0.680\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.685\n",
      "bordersmote         0.678\n",
      "svmsmote            0.677\n",
      "\n",
      "\n",
      "Name model:      \tgb\n",
      "Sampling method: undersampling + oversampling\n",
      "Best rand_ratio: \t0.35\n",
      "Best score:      \t0.7003003236846698\n",
      "Dataset size:    \t2884, Counter({0.0: 1442, 1.0: 1442})\n",
      "Combination:     \t_random_smote_gb\n",
      "\n",
      "\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "         0.0      0.936     0.889     0.912      1442\n",
      "         1.0      0.894     0.940     0.916      1442\n",
      "\n",
      "    accuracy                          0.914      2884\n",
      "   macro avg      0.915     0.914     0.914      2884\n",
      "weighted avg      0.915     0.914     0.914      2884\n",
      "\n",
      "auc macro 0.972\n",
      "confusion matrix\n",
      "[[1282  160]\n",
      " [  87 1355]]\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "         0.0      0.453     0.523     0.486       149\n",
      "         1.0      0.932     0.912     0.922      1072\n",
      "\n",
      "    accuracy                          0.865      1221\n",
      "   macro avg      0.693     0.718     0.704      1221\n",
      "weighted avg      0.874     0.865     0.869      1221\n",
      "\n",
      "auc macro 0.840\n",
      "confusion matrix\n",
      "[[ 78  71]\n",
      " [ 94 978]]\n",
      "-------------------------------------------------\n",
      "Oversample          Score               Random 0.2 Name xgb\n",
      "-------------------------------------------------\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.680\n",
      "bordersmote         0.672\n",
      "svmsmote            0.671\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.664\n",
      "bordersmote         0.674\n",
      "svmsmote            0.648\n",
      "-------------------------------------------------\n",
      "Oversample          Score               Random 0.25 Name xgb\n",
      "-------------------------------------------------\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.678\n",
      "bordersmote         0.678\n",
      "svmsmote            0.667\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.647\n",
      "bordersmote         0.657\n",
      "svmsmote            0.641\n",
      "-------------------------------------------------\n",
      "Oversample          Score               Random 0.3 Name xgb\n",
      "-------------------------------------------------\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.666\n",
      "bordersmote         0.672\n",
      "svmsmote            0.670\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.637\n",
      "bordersmote         0.653\n",
      "svmsmote            0.632\n",
      "-------------------------------------------------\n",
      "Oversample          Score               Random 0.35 Name xgb\n",
      "-------------------------------------------------\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.677\n",
      "bordersmote         0.688\n",
      "svmsmote            0.675\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.636\n",
      "bordersmote         0.621\n",
      "svmsmote            0.631\n",
      "-------------------------------------------------\n",
      "Oversample          Score               Random 0.4 Name xgb\n",
      "-------------------------------------------------\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.687\n",
      "bordersmote         0.671\n",
      "svmsmote            0.678\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.615\n",
      "bordersmote         0.613\n",
      "svmsmote            0.610\n",
      "-------------------------------------------------\n",
      "Oversample          Score               Random 0.45 Name xgb\n",
      "-------------------------------------------------\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.678\n",
      "bordersmote         0.669\n",
      "svmsmote            0.680\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.624\n",
      "bordersmote         0.604\n",
      "svmsmote            0.607\n",
      "-------------------------------------------------\n",
      "Oversample          Score               Random 0.5 Name xgb\n",
      "-------------------------------------------------\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.680\n",
      "bordersmote         0.660\n",
      "svmsmote            0.678\n",
      "########## Changing sampling mode ##########\n",
      "smote               0.603\n",
      "bordersmote         0.605\n",
      "svmsmote            0.608\n",
      "\n",
      "\n",
      "Name model:      \txgb\n",
      "Sampling method: only oversampling\n",
      "Best rand_ratio: \t0.5\n",
      "Best score:      \t0.691023166023166\n",
      "Dataset size:    \t6988, Counter({1.0: 3494, 0.0: 3494})\n",
      "Combination:     \t_random_smote_xgb\n",
      "\n",
      "\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "         0.0      0.867     0.947     0.905      3494\n",
      "         1.0      0.942     0.855     0.896      3494\n",
      "\n",
      "    accuracy                          0.901      6988\n",
      "   macro avg      0.904     0.901     0.901      6988\n",
      "weighted avg      0.904     0.901     0.901      6988\n",
      "\n",
      "auc macro 0.976\n",
      "confusion matrix\n",
      "[[3310  184]\n",
      " [ 508 2986]]\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "         0.0      0.398     0.591     0.476       149\n",
      "         1.0      0.939     0.876     0.906      1072\n",
      "\n",
      "    accuracy                          0.841      1221\n",
      "   macro avg      0.669     0.733     0.691      1221\n",
      "weighted avg      0.873     0.841     0.854      1221\n",
      "\n",
      "auc macro 0.836\n",
      "confusion matrix\n",
      "[[ 88  61]\n",
      " [133 939]]\n"
     ]
    }
   ],
   "source": [
    "from imblearn.over_sampling import SMOTE, BorderlineSMOTE, SVMSMOTE\n",
    "from collections import Counter\n",
    "from train import evaluate\n",
    "from utils import datasetSampler\n",
    "from functools import partial\n",
    "from train import evaluate\n",
    "from contextlib import redirect_stdout\n",
    "# Over sample to 50/\n",
    "overs = [\n",
    "    (\"smote\", SMOTE(sampling_strategy=1.0, k_neighbors=1)),\n",
    "    (\"bordersmote\", BorderlineSMOTE(sampling_strategy=1.0, k_neighbors=1)),\n",
    "    (\"svmsmote\", SVMSMOTE(sampling_strategy=1.0, k_neighbors=1)), \n",
    "    # ADASYN(sampling_strategy=1.0, n_neighbors=1)\n",
    "]\n",
    "\n",
    "sampler = partial(\n",
    "    datasetSampler,\n",
    "    # missing parameters: model, overSampler, sampling_strategy\n",
    "    X_train=X_train,\n",
    "    y_train=y_train,\n",
    "    X_valid=X_valid,\n",
    "    y_valid=y_valid,\n",
    "    useUnderSampler = False\n",
    ")\n",
    "random_ratio = [0.2, 0.25, 0.3, 0.35, 0.4, 0.45, 0.5]\n",
    "name_models = [\"lr\", \"svc\", \"knn\", \"rf\", \"adaboost\", \"nn\", \"gb\", \"xgb\"]\n",
    "for name in name_models:\n",
    "    best_k = 0\n",
    "    best_over = \"\"\n",
    "    best_score = 0\n",
    "    best_random = 0\n",
    "    best_model = None\n",
    "    file_name = \"\"\n",
    "    sampling_method = \"\"\n",
    "    best_X, best_y = None, None\n",
    "\n",
    "    for r in random_ratio:\n",
    "        print(\"-------------------------------------------------\")\n",
    "        print(f\"{'Oversample':<20}{'Score':<20}Random {r} Name {name}\")\n",
    "        print(\"-------------------------------------------------\")\n",
    "        for underSampler in range(1,3):\n",
    "            print(f\"########## Changing sampling mode ##########\")\n",
    "            for name_over, over in overs:\n",
    "                for k in range(2, 5):\n",
    "                    model = load(path_models+f\"{name}.joblib\")\n",
    "                    over.set_params(k_neighbors=k)\n",
    "                    result = sampler(\n",
    "                        model_name= name,\n",
    "                        model = model,\n",
    "                        overSampler = over,\n",
    "                        sampling_strategy = r,\n",
    "                        useUnderSampler = True if underSampler == 2 else False\n",
    "                    )\n",
    "                    score = result[0]\n",
    "                    X_train_sample = result[1]\n",
    "                    y_train_sample = result[2]\n",
    "                    model = result[3]\n",
    "                    if score > best_score:\n",
    "                        best_k = k\n",
    "                        best_over = name_over\n",
    "                        best_score = score\n",
    "                        best_random = r\n",
    "                        best_model = copy.deepcopy(model)\n",
    "                        sampling_method = \"only oversampling\" if underSampler == 1 else \"undersampling + oversampling\"\n",
    "                        suffix = \"_only_oversampling\" if underSampler == 1 else \"_sampling\"\n",
    "                        file_name = output_models.replace(\"models_output/\", '').replace(\"/\",\"\") + suffix + \".txt\"\n",
    "                        best_X = X_train_sample\n",
    "                        best_y = y_train_sample\n",
    "                #in this case the score is the last score computed with k = 4\n",
    "                print(f'{name_over:<20}{score:.3f}')\n",
    "\n",
    "    # Save the dataset and model\n",
    "    combination = f\"_random_{best_over}_{name}\"\n",
    "    tmp = np.concatenate((best_X, np.expand_dims(best_y, 1)), axis=1)\n",
    "    tmp = pd.DataFrame(tmp, columns=feat_names)\n",
    "\n",
    "    # Current model's statistics\n",
    "    print(\"\\n\")\n",
    "    print(f\"Name model:      \\t{name}\")\n",
    "    print(f\"Sampling method: {sampling_method }\")\n",
    "    print(f\"Best rand_ratio: \\t{best_random}\")\n",
    "    print(f\"Best score:      \\t{best_score}\")\n",
    "    print(f\"Dataset size:    \\t{len(best_y)}, {Counter(best_y)}\")\n",
    "    print(f\"Combination:     \\t{combination}\")\n",
    "    print(\"\\n\\n\")\n",
    "\n",
    "    # Evaluate the best model, save the data and the best model\n",
    "    evaluate(best_model, best_X, best_y)\n",
    "    evaluate(best_model, X_valid, y_valid)\n",
    "    tmp.to_csv(path+f\"train{combination}.csv\")\n",
    "    dump(best_model, path_models+f\"{name}{combination}.joblib\")\n",
    "    with open(f\"{output_models}{file_name}\", 'a+') as f:\n",
    "            with redirect_stdout(f):\n",
    "                print (f\"####################   {name}    #########################\")\n",
    "                print(\"Testing on training set:\")\n",
    "                evaluate(best_model, best_X, best_y)\n",
    "                print(\"Testing on validation set:\")\n",
    "                evaluate(best_model, X_valid, y_valid)\n",
    "                print(\"Parameters: {0}\".format(model.get_params()))\n",
    "                print (f\"####################   {name}  END   #########################\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}