ensemble_pipeline.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.metrics import accuracy_score, roc_auc_score, recall_score\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from imblearn.over_sampling import SMOTE\n",
    "from imblearn.pipeline import Pipeline\n",
    "\n",
    "from collections import Counter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.set_option('display.max_columns', None)\n",
    "pd.set_option('display.max_rows', None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "random_state = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train = pd.read_csv('dataset_diabetes/diabetic_data_train.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Removing several medication columns also\n",
    "remove_cols = ['encounter_id', 'patient_nbr', 'weight', 'payer_code', 'medical_specialty', 'examide', \n",
    "    'citoglipton', 'chlorpropamide', 'acetohexamide', 'tolbutamide', 'acarbose', 'miglitol', \n",
    "    'troglitazone', 'tolazamide', 'glyburide-metformin', 'glipizide-metformin', 'glimepiride-pioglitazone', \n",
    "    'metformin-rosiglitazone', 'metformin-pioglitazone']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train = df_train[[col for col in df_train.columns if col not in remove_cols]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['race', 'gender', 'age', 'admission_type_id',\n",
       "       'discharge_disposition_id', 'admission_source_id', 'time_in_hospital',\n",
       "       'num_lab_procedures', 'num_procedures', 'num_medications',\n",
       "       'number_outpatient', 'number_emergency', 'number_inpatient', 'diag_1',\n",
       "       'diag_2', 'diag_3', 'number_diagnoses', 'max_glu_serum', 'A1Cresult',\n",
       "       'metformin', 'repaglinide', 'nateglinide', 'glimepiride', 'glipizide',\n",
       "       'glyburide', 'pioglitazone', 'rosiglitazone', 'insulin', 'change',\n",
       "       'diabetesMed', 'readmitted'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.columns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create Labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('NO', 0.5393799439886012),\n",
       " ('>30', 0.3491868520611212),\n",
       " ('<30', 0.1114332039502776)]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[(key, val/len(df_train.readmitted)) for (key, val) in Counter(df_train.readmitted).most_common()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def binarize_readmitted(x):\n",
    "    if x in ['<30']: # readmitted in less than 30 days is positive class\n",
    "        return 1 \n",
    "    else:\n",
    "        return 0 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train['label'] = df_train.apply(lambda x: binarize_readmitted(x['readmitted']), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0, 0.8885667960497224), (1, 0.1114332039502776)]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[(key, val/len(df_train.readmitted)) for (key, val) in Counter(df_train['label']).most_common()]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Process Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def map_age(x):\n",
    "    if x == '[0-10)':\n",
    "        return 5\n",
    "    elif x == '[10-20)':\n",
    "        return 15\n",
    "    elif x == '[20-30)':\n",
    "        return 25\n",
    "    elif x == '[30-40)':\n",
    "        return 35\n",
    "    elif x == '[40-50)':\n",
    "        return 45\n",
    "    elif x == '[50-60)':\n",
    "        return 55\n",
    "    elif x == '[60-70)':\n",
    "        return 65\n",
    "    elif x == '[70-80)':\n",
    "        return 75\n",
    "    elif x == '[80-90)':\n",
    "        return 85\n",
    "    else:\n",
    "        return 95\n",
    "\n",
    "admission_dict = {1: 'Emergency', 2: 'Urgent', 3: 'Elective', 4: 'Newborn', 7: 'Trauma Center'}\n",
    "def map_admission(x):\n",
    "    if x in [5,6,8]:\n",
    "        return 'Not Available'\n",
    "    else:\n",
    "        return admission_dict[x]\n",
    "    \n",
    "def discharged_home(x):\n",
    "    if x == 1:\n",
    "        return 1 # Discharged home\n",
    "    else:\n",
    "        return 0 # Other \n",
    "\n",
    "def map_source(x):\n",
    "    if x in [1,2,3]:\n",
    "        return 'Referral' \n",
    "    elif x == 7:\n",
    "        return 'Emergency Room'\n",
    "    else:\n",
    "        return 'Other'\n",
    "\n",
    "def map_diag(x):\n",
    "    if x[0].isdigit() == False:\n",
    "        return 'other'\n",
    "    \n",
    "    float_x = float(x)\n",
    "    if (float_x >= 390 and float_x <= 459) or float_x == 785: # 390–459, 785\n",
    "        return 'circulatory'\n",
    "    elif (float_x >= 460 and float_x <= 519) or float_x == 786: # 460–519, 786\n",
    "        return 'respiratory'\n",
    "    elif (float_x >= 520 and float_x <= 579) or float_x == 787: # 520–579, 787\n",
    "        return 'digestive'\n",
    "    elif float_x >= 250 and float_x < 251: # 250.xx\n",
    "        return 'diabetes'\n",
    "    elif float_x >= 800 and float_x <= 999: # 800–999\n",
    "        return 'injury'\n",
    "    elif float_x >= 710 and float_x <= 739: # 710–739\n",
    "        return 'musculoskeletal'\n",
    "    elif (float_x >= 580 and float_x <= 629) or float_x == 788: # 580–629, 788\n",
    "        return 'genitourinary'\n",
    "    elif float_x >= 140 and float_x <= 239: # 140–239\n",
    "        return 'neoplasms'\n",
    "    else:\n",
    "        return 'other'\n",
    "\n",
    "def binarize_yn(x):\n",
    "    if x in ['Yes', 'Ch']: \n",
    "        return 1 \n",
    "    else:\n",
    "        return 0 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train['age_processed'] = df_train.apply(lambda x: map_age(x['age']), axis=1)\n",
    "df_train['admission_type_processed'] = df_train.apply(lambda x: map_admission(x['admission_type_id']), axis=1)\n",
    "df_train['discharged_processed'] = df_train.apply(lambda x: discharged_home(x['discharge_disposition_id']), axis=1)\n",
    "df_train['source_processed'] = df_train.apply(lambda x: map_source(x['admission_source_id']), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train['diag1_processed'] = df_train.apply(lambda x: map_diag(x['diag_1']), axis=1)\n",
    "df_train['diag2_processed'] = df_train.apply(lambda x: map_diag(x['diag_2']), axis=1)\n",
    "df_train['diag3_processed'] = df_train.apply(lambda x: map_diag(x['diag_3']), axis=1)\n",
    "df_train['change_processed'] = df_train.apply(lambda x: binarize_yn(x['change']), axis=1)\n",
    "df_train['diabetesMed_processed'] = df_train.apply(lambda x: binarize_yn(x['diabetesMed']), axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## One Hot Encodings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "ohe_cols = ['admission_type_processed', 'source_processed',\n",
    "    'diag1_processed', 'diag2_processed', 'diag3_processed', 'race', 'gender', \n",
    "    'max_glu_serum', 'A1Cresult', 'metformin', 'repaglinide', 'nateglinide', \n",
    "    'glimepiride', 'glipizide', 'glyburide', 'pioglitazone', 'rosiglitazone', 'insulin']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Inspecting values/counts for each of the categorical variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "admission_type_processed\n",
      "Counter({'Emergency': 43218, 'Elective': 15052, 'Urgent': 14845, 'Not Available': 8274, 'Trauma Center': 13, 'Newborn': 10}) \n",
      "\n",
      "source_processed\n",
      "Counter({'Emergency Room': 45990, 'Referral': 24668, 'Other': 10754}) \n",
      "\n",
      "diag1_processed\n",
      "Counter({'circulatory': 24429, 'other': 14636, 'respiratory': 11530, 'digestive': 7549, 'diabetes': 6987, 'injury': 5491, 'genitourinary': 4122, 'musculoskeletal': 3929, 'neoplasms': 2739}) \n",
      "\n",
      "diag2_processed\n",
      "Counter({'circulatory': 25456, 'other': 21617, 'diabetes': 10217, 'respiratory': 8679, 'genitourinary': 6717, 'digestive': 3318, 'neoplasms': 2039, 'injury': 1971, 'musculoskeletal': 1398}) \n",
      "\n",
      "diag3_processed\n",
      "Counter({'other': 24576, 'circulatory': 24159, 'diabetes': 13759, 'respiratory': 5865, 'genitourinary': 5354, 'digestive': 3129, 'injury': 1560, 'musculoskeletal': 1517, 'neoplasms': 1493}) \n",
      "\n",
      "race\n",
      "Counter({'Caucasian': 60871, 'AfricanAmerican': 15325, '?': 1824, 'Hispanic': 1650, 'Other': 1225, 'Asian': 517}) \n",
      "\n",
      "gender\n",
      "Counter({'Female': 43740, 'Male': 37670, 'Unknown/Invalid': 2}) \n",
      "\n",
      "max_glu_serum\n",
      "Counter({'None': 77130, 'Norm': 2068, '>200': 1201, '>300': 1013}) \n",
      "\n",
      "A1Cresult\n",
      "Counter({'None': 67778, '>8': 6600, 'Norm': 4043, '>7': 2991}) \n",
      "\n",
      "metformin\n",
      "Counter({'No': 65384, 'Steady': 14704, 'Up': 869, 'Down': 455}) \n",
      "\n",
      "repaglinide\n",
      "Counter({'No': 80162, 'Steady': 1126, 'Up': 89, 'Down': 35}) \n",
      "\n",
      "nateglinide\n",
      "Counter({'No': 80845, 'Steady': 539, 'Up': 20, 'Down': 8}) \n",
      "\n",
      "glimepiride\n",
      "Counter({'No': 77255, 'Steady': 3738, 'Up': 266, 'Down': 153}) \n",
      "\n",
      "glipizide\n",
      "Counter({'No': 71346, 'Steady': 8984, 'Up': 623, 'Down': 459}) \n",
      "\n",
      "glyburide\n",
      "Counter({'No': 72905, 'Steady': 7406, 'Up': 645, 'Down': 456}) \n",
      "\n",
      "pioglitazone\n",
      "Counter({'No': 75594, 'Steady': 5549, 'Up': 180, 'Down': 89}) \n",
      "\n",
      "rosiglitazone\n",
      "Counter({'No': 76324, 'Steady': 4876, 'Up': 144, 'Down': 68}) \n",
      "\n",
      "insulin\n",
      "Counter({'No': 37931, 'Steady': 24592, 'Down': 9793, 'Up': 9096}) \n",
      "\n"
     ]
    }
   ],
   "source": [
    "for col in ohe_cols:\n",
    "    print(col)\n",
    "    print(Counter(df_train[col]), '\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "81410\n"
     ]
    }
   ],
   "source": [
    "# Remove 2 rows where gender is invalid (probably doesn't matter but just incase)\n",
    "df_train = df_train[df_train.gender != 'Unknown/Invalid']\n",
    "print(len(df_train))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train_sub = df_train[ohe_cols]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "ohe = OneHotEncoder(categories='auto')\n",
    "train_ohe = ohe.fit_transform(df_train_sub).toarray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# obtain column names for one hot encoded columns \n",
    "feature_labels = []\n",
    "feature_labels_ = ohe.categories_\n",
    "for col, values in zip(df_train_sub.columns, feature_labels_):\n",
    "    for val in values:\n",
    "        feature_labels.append(col+'_'+val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train_ohe = pd.DataFrame(train_ohe, columns = feature_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "cols = ['age_processed', 'discharged_processed', 'change_processed', 'diabetesMed_processed', \n",
    "        'time_in_hospital', 'num_lab_procedures', 'num_procedures', 'num_medications', \n",
    "        'number_outpatient','number_emergency', 'number_inpatient', 'number_diagnoses']\n",
    "\n",
    "df_train_num = df_train[cols]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(81410, 88) (81410, 12)\n"
     ]
    }
   ],
   "source": [
    "print(df_train_ohe.shape, df_train_num.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Reindexing because df_train_ohe has different indices than df_train_num (due to removing the two invalid gender entries)\n",
    "# and not doing this will create null entries when concatenating \n",
    "df_train_ohe.reset_index(drop=True, inplace=True)\n",
    "df_train_num.reset_index(drop=True, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = pd.concat([df_train_ohe, df_train_num], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "admission_type_processed_Elective\n",
      "Counter({0.0: 66358, 1.0: 15052}) \n",
      "\n",
      "admission_type_processed_Emergency\n",
      "Counter({1.0: 43216, 0.0: 38194}) \n",
      "\n",
      "admission_type_processed_Newborn\n",
      "Counter({0.0: 81400, 1.0: 10}) \n",
      "\n",
      "admission_type_processed_Not Available\n",
      "Counter({0.0: 73136, 1.0: 8274}) \n",
      "\n",
      "admission_type_processed_Trauma Center\n",
      "Counter({0.0: 81397, 1.0: 13}) \n",
      "\n",
      "admission_type_processed_Urgent\n",
      "Counter({0.0: 66565, 1.0: 14845}) \n",
      "\n",
      "source_processed_Emergency Room\n",
      "Counter({1.0: 45988, 0.0: 35422}) \n",
      "\n",
      "source_processed_Other\n",
      "Counter({0.0: 70656, 1.0: 10754}) \n",
      "\n",
      "source_processed_Referral\n",
      "Counter({0.0: 56742, 1.0: 24668}) \n",
      "\n",
      "diag1_processed_circulatory\n",
      "Counter({0.0: 56981, 1.0: 24429}) \n",
      "\n",
      "diag1_processed_diabetes\n",
      "Counter({0.0: 74423, 1.0: 6987}) \n",
      "\n",
      "diag1_processed_digestive\n",
      "Counter({0.0: 73861, 1.0: 7549}) \n",
      "\n",
      "diag1_processed_genitourinary\n",
      "Counter({0.0: 77288, 1.0: 4122}) \n",
      "\n",
      "diag1_processed_injury\n",
      "Counter({0.0: 75921, 1.0: 5489}) \n",
      "\n",
      "diag1_processed_musculoskeletal\n",
      "Counter({0.0: 77481, 1.0: 3929}) \n",
      "\n",
      "diag1_processed_neoplasms\n",
      "Counter({0.0: 78671, 1.0: 2739}) \n",
      "\n",
      "diag1_processed_other\n",
      "Counter({0.0: 66774, 1.0: 14636}) \n",
      "\n",
      "diag1_processed_respiratory\n",
      "Counter({0.0: 69880, 1.0: 11530}) \n",
      "\n",
      "diag2_processed_circulatory\n",
      "Counter({0.0: 55954, 1.0: 25456}) \n",
      "\n",
      "diag2_processed_diabetes\n",
      "Counter({0.0: 71193, 1.0: 10217}) \n",
      "\n",
      "diag2_processed_digestive\n",
      "Counter({0.0: 78092, 1.0: 3318}) \n",
      "\n",
      "diag2_processed_genitourinary\n",
      "Counter({0.0: 74693, 1.0: 6717}) \n",
      "\n",
      "diag2_processed_injury\n",
      "Counter({0.0: 79441, 1.0: 1969}) \n",
      "\n",
      "diag2_processed_musculoskeletal\n",
      "Counter({0.0: 80012, 1.0: 1398}) \n",
      "\n",
      "diag2_processed_neoplasms\n",
      "Counter({0.0: 79371, 1.0: 2039}) \n",
      "\n",
      "diag2_processed_other\n",
      "Counter({0.0: 59793, 1.0: 21617}) \n",
      "\n",
      "diag2_processed_respiratory\n",
      "Counter({0.0: 72731, 1.0: 8679}) \n",
      "\n",
      "diag3_processed_circulatory\n",
      "Counter({0.0: 57251, 1.0: 24159}) \n",
      "\n",
      "diag3_processed_diabetes\n",
      "Counter({0.0: 67651, 1.0: 13759}) \n",
      "\n",
      "diag3_processed_digestive\n",
      "Counter({0.0: 78281, 1.0: 3129}) \n",
      "\n",
      "diag3_processed_genitourinary\n",
      "Counter({0.0: 76056, 1.0: 5354}) \n",
      "\n",
      "diag3_processed_injury\n",
      "Counter({0.0: 79851, 1.0: 1559}) \n",
      "\n",
      "diag3_processed_musculoskeletal\n",
      "Counter({0.0: 79893, 1.0: 1517}) \n",
      "\n",
      "diag3_processed_neoplasms\n",
      "Counter({0.0: 79917, 1.0: 1493}) \n",
      "\n",
      "diag3_processed_other\n",
      "Counter({0.0: 56835, 1.0: 24575}) \n",
      "\n",
      "diag3_processed_respiratory\n",
      "Counter({0.0: 75545, 1.0: 5865}) \n",
      "\n",
      "race_?\n",
      "Counter({0.0: 79588, 1.0: 1822}) \n",
      "\n",
      "race_AfricanAmerican\n",
      "Counter({0.0: 66085, 1.0: 15325}) \n",
      "\n",
      "race_Asian\n",
      "Counter({0.0: 80893, 1.0: 517}) \n",
      "\n",
      "race_Caucasian\n",
      "Counter({1.0: 60871, 0.0: 20539}) \n",
      "\n",
      "race_Hispanic\n",
      "Counter({0.0: 79760, 1.0: 1650}) \n",
      "\n",
      "race_Other\n",
      "Counter({0.0: 80185, 1.0: 1225}) \n",
      "\n",
      "gender_Female\n",
      "Counter({1.0: 43740, 0.0: 37670}) \n",
      "\n",
      "gender_Male\n",
      "Counter({0.0: 43740, 1.0: 37670}) \n",
      "\n",
      "max_glu_serum_>200\n",
      "Counter({0.0: 80209, 1.0: 1201}) \n",
      "\n",
      "max_glu_serum_>300\n",
      "Counter({0.0: 80397, 1.0: 1013}) \n",
      "\n",
      "max_glu_serum_None\n",
      "Counter({1.0: 77128, 0.0: 4282}) \n",
      "\n",
      "max_glu_serum_Norm\n",
      "Counter({0.0: 79342, 1.0: 2068}) \n",
      "\n",
      "A1Cresult_>7\n",
      "Counter({0.0: 78419, 1.0: 2991}) \n",
      "\n",
      "A1Cresult_>8\n",
      "Counter({0.0: 74810, 1.0: 6600}) \n",
      "\n",
      "A1Cresult_None\n",
      "Counter({1.0: 67776, 0.0: 13634}) \n",
      "\n",
      "A1Cresult_Norm\n",
      "Counter({0.0: 77367, 1.0: 4043}) \n",
      "\n",
      "metformin_Down\n",
      "Counter({0.0: 80955, 1.0: 455}) \n",
      "\n",
      "metformin_No\n",
      "Counter({1.0: 65383, 0.0: 16027}) \n",
      "\n",
      "metformin_Steady\n",
      "Counter({0.0: 66707, 1.0: 14703}) \n",
      "\n",
      "metformin_Up\n",
      "Counter({0.0: 80541, 1.0: 869}) \n",
      "\n",
      "repaglinide_Down\n",
      "Counter({0.0: 81375, 1.0: 35}) \n",
      "\n",
      "repaglinide_No\n",
      "Counter({1.0: 80160, 0.0: 1250}) \n",
      "\n",
      "repaglinide_Steady\n",
      "Counter({0.0: 80284, 1.0: 1126}) \n",
      "\n",
      "repaglinide_Up\n",
      "Counter({0.0: 81321, 1.0: 89}) \n",
      "\n",
      "nateglinide_Down\n",
      "Counter({0.0: 81402, 1.0: 8}) \n",
      "\n",
      "nateglinide_No\n",
      "Counter({1.0: 80843, 0.0: 567}) \n",
      "\n",
      "nateglinide_Steady\n",
      "Counter({0.0: 80871, 1.0: 539}) \n",
      "\n",
      "nateglinide_Up\n",
      "Counter({0.0: 81390, 1.0: 20}) \n",
      "\n",
      "glimepiride_Down\n",
      "Counter({0.0: 81257, 1.0: 153}) \n",
      "\n",
      "glimepiride_No\n",
      "Counter({1.0: 77253, 0.0: 4157}) \n",
      "\n",
      "glimepiride_Steady\n",
      "Counter({0.0: 77672, 1.0: 3738}) \n",
      "\n",
      "glimepiride_Up\n",
      "Counter({0.0: 81144, 1.0: 266}) \n",
      "\n",
      "glipizide_Down\n",
      "Counter({0.0: 80951, 1.0: 459}) \n",
      "\n",
      "glipizide_No\n",
      "Counter({1.0: 71344, 0.0: 10066}) \n",
      "\n",
      "glipizide_Steady\n",
      "Counter({0.0: 72426, 1.0: 8984}) \n",
      "\n",
      "glipizide_Up\n",
      "Counter({0.0: 80787, 1.0: 623}) \n",
      "\n",
      "glyburide_Down\n",
      "Counter({0.0: 80954, 1.0: 456}) \n",
      "\n",
      "glyburide_No\n",
      "Counter({1.0: 72903, 0.0: 8507}) \n",
      "\n",
      "glyburide_Steady\n",
      "Counter({0.0: 74004, 1.0: 7406}) \n",
      "\n",
      "glyburide_Up\n",
      "Counter({0.0: 80765, 1.0: 645}) \n",
      "\n",
      "pioglitazone_Down\n",
      "Counter({0.0: 81321, 1.0: 89}) \n",
      "\n",
      "pioglitazone_No\n",
      "Counter({1.0: 75592, 0.0: 5818}) \n",
      "\n",
      "pioglitazone_Steady\n",
      "Counter({0.0: 75861, 1.0: 5549}) \n",
      "\n",
      "pioglitazone_Up\n",
      "Counter({0.0: 81230, 1.0: 180}) \n",
      "\n",
      "rosiglitazone_Down\n",
      "Counter({0.0: 81342, 1.0: 68}) \n",
      "\n",
      "rosiglitazone_No\n",
      "Counter({1.0: 76323, 0.0: 5087}) \n",
      "\n",
      "rosiglitazone_Steady\n",
      "Counter({0.0: 76535, 1.0: 4875}) \n",
      "\n",
      "rosiglitazone_Up\n",
      "Counter({0.0: 81266, 1.0: 144}) \n",
      "\n",
      "insulin_Down\n",
      "Counter({0.0: 71617, 1.0: 9793}) \n",
      "\n",
      "insulin_No\n",
      "Counter({0.0: 43481, 1.0: 37929}) \n",
      "\n",
      "insulin_Steady\n",
      "Counter({0.0: 56818, 1.0: 24592}) \n",
      "\n",
      "insulin_Up\n",
      "Counter({0.0: 72314, 1.0: 9096}) \n",
      "\n",
      "age_processed\n",
      "Counter({75: 20878, 65: 17958, 55: 13741, 85: 13737, 45: 7810, 35: 3030, 95: 2239, 25: 1335, 15: 551, 5: 131}) \n",
      "\n",
      "discharged_processed\n",
      "Counter({1: 48287, 0: 33123}) \n",
      "\n",
      "change_processed\n",
      "Counter({0: 43754, 1: 37656}) \n",
      "\n",
      "diabetesMed_processed\n",
      "Counter({1: 62616, 0: 18794}) \n",
      "\n",
      "time_in_hospital\n",
      "Counter({3: 14169, 2: 13819, 1: 11375, 4: 11087, 5: 7926, 6: 6033, 7: 4704, 8: 3504, 9: 2425, 10: 1890, 11: 1476, 12: 1157, 13: 1004, 14: 841}) \n",
      "\n",
      "num_lab_procedures\n",
      "Counter({1: 2598, 43: 2208, 44: 1984, 45: 1905, 38: 1785, 40: 1775, 46: 1725, 37: 1695, 41: 1685, 47: 1674, 42: 1661, 48: 1647, 39: 1647, 49: 1630, 36: 1587, 35: 1541, 50: 1538, 51: 1528, 54: 1509, 52: 1506, 56: 1463, 55: 1460, 53: 1439, 57: 1400, 58: 1370, 34: 1368, 61: 1318, 60: 1286, 59: 1278, 63: 1136, 62: 1130, 64: 1091, 30: 1077, 29: 1060, 31: 1047, 32: 1045, 33: 1024, 66: 984, 65: 972, 2: 901, 26: 875, 67: 872, 25: 848, 68: 841, 28: 824, 69: 794, 19: 769, 9: 754, 27: 680, 10: 675, 70: 666, 23: 657, 71: 614, 20: 605, 17: 605, 72: 604, 24: 602, 22: 582, 18: 572, 11: 543, 21: 537, 3: 533, 73: 520, 74: 503, 16: 433, 75: 431, 12: 407, 76: 377, 15: 363, 13: 343, 77: 338, 4: 315, 78: 293, 8: 283, 14: 266, 7: 266, 79: 260, 80: 252, 5: 233, 81: 225, 6: 218, 83: 191, 82: 185, 84: 127, 85: 126, 86: 106, 88: 82, 87: 74, 89: 62, 90: 51, 91: 50, 93: 47, 92: 41, 94: 39, 95: 38, 97: 27, 98: 20, 96: 19, 101: 11, 100: 9, 99: 8, 102: 7, 103: 5, 106: 5, 108: 4, 109: 4, 105: 4, 104: 3, 114: 2, 121: 1, 107: 1, 120: 1, 132: 1, 113: 1, 118: 1, 111: 1, 129: 1}) \n",
      "\n",
      "num_procedures\n",
      "Counter({0: 37349, 1: 16624, 2: 10131, 3: 7534, 6: 3928, 4: 3341, 5: 2503}) \n",
      "\n",
      "num_medications\n",
      "Counter({13: 4879, 12: 4816, 11: 4669, 15: 4611, 14: 4593, 16: 4351, 10: 4259, 17: 3945, 9: 3903, 18: 3626, 8: 3511, 19: 3289, 20: 2976, 7: 2753, 21: 2547, 22: 2282, 6: 2196, 23: 1906, 24: 1698, 5: 1618, 25: 1504, 26: 1290, 4: 1136, 27: 1132, 28: 966, 29: 780, 3: 716, 30: 685, 31: 570, 32: 503, 33: 386, 2: 363, 34: 355, 35: 317, 36: 238, 37: 234, 1: 219, 38: 196, 39: 165, 40: 150, 41: 109, 43: 108, 42: 97, 46: 76, 45: 74, 44: 72, 47: 61, 49: 50, 48: 50, 50: 45, 52: 44, 51: 33, 53: 29, 56: 29, 54: 27, 55: 25, 57: 23, 58: 20, 59: 15, 61: 13, 60: 13, 63: 12, 62: 10, 65: 9, 68: 6, 67: 5, 66: 5, 64: 4, 72: 3, 69: 3, 75: 2, 70: 2, 81: 1, 79: 1, 74: 1}) \n",
      "\n",
      "number_outpatient\n",
      "Counter({0: 68018, 1: 6865, 2: 2872, 3: 1620, 4: 888, 5: 423, 6: 228, 7: 130, 8: 72, 9: 71, 10: 41, 11: 35, 13: 24, 12: 23, 14: 22, 15: 15, 16: 13, 22: 5, 21: 5, 18: 5, 20: 5, 17: 5, 19: 3, 27: 3, 24: 2, 33: 2, 23: 2, 35: 2, 26: 2, 34: 1, 29: 1, 36: 1, 28: 1, 25: 1, 38: 1, 37: 1, 39: 1, 40: 1}) \n",
      "\n",
      "number_emergency\n",
      "Counter({0: 72264, 1: 6158, 2: 1657, 3: 582, 4: 300, 5: 147, 6: 72, 7: 61, 8: 40, 10: 28, 9: 27, 11: 19, 13: 10, 12: 8, 22: 5, 18: 4, 19: 4, 15: 3, 16: 3, 20: 3, 14: 3, 21: 2, 25: 2, 42: 1, 76: 1, 24: 1, 28: 1, 64: 1, 29: 1, 37: 1, 54: 1}) \n",
      "\n",
      "number_inpatient\n",
      "Counter({0: 54100, 1: 15646, 2: 6047, 3: 2691, 4: 1285, 5: 661, 6: 388, 7: 220, 8: 119, 9: 94, 10: 50, 11: 40, 12: 29, 13: 16, 14: 9, 15: 7, 16: 4, 19: 2, 17: 1, 18: 1}) \n",
      "\n",
      "number_diagnoses\n",
      "Counter({9: 39570, 5: 9119, 8: 8494, 7: 8301, 6: 8121, 4: 4446, 3: 2280, 2: 818, 1: 168, 16: 37, 10: 14, 13: 13, 12: 8, 11: 8, 14: 7, 15: 6}) \n",
      "\n"
     ]
    }
   ],
   "source": [
    "for col in X_train.columns:\n",
    "    print(col)\n",
    "    print(Counter(X_train[col]), '\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = df_train['label'].values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Cross Validation Pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['memory', 'steps', 'verbose', 'sampling', 'model', 'sampling__k_neighbors', 'sampling__n_jobs', 'sampling__random_state', 'sampling__sampling_strategy', 'model__bootstrap', 'model__ccp_alpha', 'model__class_weight', 'model__criterion', 'model__max_depth', 'model__max_features', 'model__max_leaf_nodes', 'model__max_samples', 'model__min_impurity_decrease', 'model__min_impurity_split', 'model__min_samples_leaf', 'model__min_samples_split', 'model__min_weight_fraction_leaf', 'model__n_estimators', 'model__n_jobs', 'model__oob_score', 'model__random_state', 'model__verbose', 'model__warm_start'])"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipe = Pipeline([\n",
    "        ('sampling', SMOTE(random_state=random_state)),\n",
    "        ('model', RandomForestClassifier())\n",
    "    ])\n",
    "\n",
    "pipe.get_params().keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "param_grid = {\n",
    "    'model__n_estimators': [100, 500],\n",
    "    'model__max_depth': [4, 6, 8, 10, 12],\n",
    "    'model__random_state': [random_state]\n",
    "}\n",
    "\n",
    "# Recall = TP/TP+FN -> Of the ones that are positive, how many did we indicate as positive\n",
    "# AUROC = area under curve TPR (TP/TP+FN) vs. FPR (FP/TN+FP) at various thresholds\n",
    "scoring = ['accuracy', 'roc_auc', 'recall']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf = GridSearchCV(pipe, param_grid, cv=4, verbose=5, scoring=scoring, refit='roc_auc', return_train_score=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 4 folds for each of 10 candidates, totalling 40 fits\n",
      "[CV] model__max_depth=4, model__n_estimators=100, model__random_state=0 \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  model__max_depth=4, model__n_estimators=100, model__random_state=0, accuracy=(train=0.846, test=0.844), recall=(train=0.085, test=0.082), roc_auc=(train=0.577, test=0.566), total=   6.0s\n",
      "[CV] model__max_depth=4, model__n_estimators=100, model__random_state=0 \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    6.6s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  model__max_depth=4, model__n_estimators=100, model__random_state=0, accuracy=(train=0.843, test=0.842), recall=(train=0.093, test=0.085), roc_auc=(train=0.578, test=0.577), total=   6.1s\n",
      "[CV] model__max_depth=4, model__n_estimators=100, model__random_state=0 \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:   13.4s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  model__max_depth=4, model__n_estimators=100, model__random_state=0, accuracy=(train=0.833, test=0.833), recall=(train=0.110, test=0.122), roc_auc=(train=0.580, test=0.566), total=   5.9s\n",
      "[CV] model__max_depth=4, model__n_estimators=100, model__random_state=0 \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:   20.0s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  model__max_depth=4, model__n_estimators=100, model__random_state=0, accuracy=(train=0.841, test=0.843), recall=(train=0.100, test=0.091), roc_auc=(train=0.580, test=0.573), total=   6.0s\n",
      "[CV] model__max_depth=4, model__n_estimators=500, model__random_state=0 \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed:   26.6s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  model__max_depth=4, model__n_estimators=500, model__random_state=0, accuracy=(train=0.842, test=0.842), recall=(train=0.092, test=0.090), roc_auc=(train=0.578, test=0.570), total=  24.8s\n",
      "[CV] model__max_depth=4, model__n_estimators=500, model__random_state=0 \n",
      "[CV]  model__max_depth=4, model__n_estimators=500, model__random_state=0, accuracy=(train=0.839, test=0.837), recall=(train=0.102, test=0.096), roc_auc=(train=0.579, test=0.575), total=  24.1s\n",
      "[CV] model__max_depth=4, model__n_estimators=500, model__random_state=0 \n",
      "[CV]  model__max_depth=4, model__n_estimators=500, model__random_state=0, accuracy=(train=0.835, test=0.833), recall=(train=0.110, test=0.112), roc_auc=(train=0.580, test=0.567), total=  25.3s\n",
      "[CV] model__max_depth=4, model__n_estimators=500, model__random_state=0 \n",
      "[CV]  model__max_depth=4, model__n_estimators=500, model__random_state=0, accuracy=(train=0.842, test=0.843), recall=(train=0.095, test=0.091), roc_auc=(train=0.580, test=0.573), total=  27.6s\n",
      "[CV] model__max_depth=6, model__n_estimators=100, model__random_state=0 \n",
      "[CV]  model__max_depth=6, model__n_estimators=100, model__random_state=0, accuracy=(train=0.880, test=0.879), recall=(train=0.022, test=0.018), roc_auc=(train=0.594, test=0.578), total=   7.6s\n",
      "[CV] model__max_depth=6, model__n_estimators=100, model__random_state=0 \n",
      "[CV]  model__max_depth=6, model__n_estimators=100, model__random_state=0, accuracy=(train=0.873, test=0.870), recall=(train=0.037, test=0.031), roc_auc=(train=0.591, test=0.581), total=   7.6s\n",
      "[CV] model__max_depth=6, model__n_estimators=100, model__random_state=0 \n",
      "[CV]  model__max_depth=6, model__n_estimators=100, model__random_state=0, accuracy=(train=0.872, test=0.871), recall=(train=0.037, test=0.041), roc_auc=(train=0.593, test=0.575), total=   7.2s\n",
      "[CV] model__max_depth=6, model__n_estimators=100, model__random_state=0 \n",
      "[CV]  model__max_depth=6, model__n_estimators=100, model__random_state=0, accuracy=(train=0.870, test=0.872), recall=(train=0.042, test=0.041), roc_auc=(train=0.597, test=0.580), total=   7.7s\n",
      "[CV] model__max_depth=6, model__n_estimators=500, model__random_state=0 \n",
      "[CV]  model__max_depth=6, model__n_estimators=500, model__random_state=0, accuracy=(train=0.879, test=0.879), recall=(train=0.024, test=0.019), roc_auc=(train=0.594, test=0.580), total=  33.0s\n",
      "[CV] model__max_depth=6, model__n_estimators=500, model__random_state=0 \n",
      "[CV]  model__max_depth=6, model__n_estimators=500, model__random_state=0, accuracy=(train=0.879, test=0.877), recall=(train=0.025, test=0.021), roc_auc=(train=0.594, test=0.582), total=  33.7s\n",
      "[CV] model__max_depth=6, model__n_estimators=500, model__random_state=0 \n",
      "[CV]  model__max_depth=6, model__n_estimators=500, model__random_state=0, accuracy=(train=0.875, test=0.875), recall=(train=0.028, test=0.029), roc_auc=(train=0.594, test=0.575), total=  32.2s\n",
      "[CV] model__max_depth=6, model__n_estimators=500, model__random_state=0 \n",
      "[CV]  model__max_depth=6, model__n_estimators=500, model__random_state=0, accuracy=(train=0.874, test=0.876), recall=(train=0.032, test=0.031), roc_auc=(train=0.596, test=0.580), total=  33.9s\n",
      "[CV] model__max_depth=8, model__n_estimators=100, model__random_state=0 \n",
      "[CV]  model__max_depth=8, model__n_estimators=100, model__random_state=0, accuracy=(train=0.886, test=0.886), recall=(train=0.007, test=0.005), roc_auc=(train=0.620, test=0.586), total=  11.0s\n",
      "[CV] model__max_depth=8, model__n_estimators=100, model__random_state=0 \n",
      "[CV]  model__max_depth=8, model__n_estimators=100, model__random_state=0, accuracy=(train=0.885, test=0.884), recall=(train=0.010, test=0.007), roc_auc=(train=0.619, test=0.589), total=   9.4s\n",
      "[CV] model__max_depth=8, model__n_estimators=100, model__random_state=0 \n",
      "[CV]  model__max_depth=8, model__n_estimators=100, model__random_state=0, accuracy=(train=0.885, test=0.886), recall=(train=0.008, test=0.007), roc_auc=(train=0.620, test=0.583), total=   9.0s\n",
      "[CV] model__max_depth=8, model__n_estimators=100, model__random_state=0 \n",
      "[CV]  model__max_depth=8, model__n_estimators=100, model__random_state=0, accuracy=(train=0.886, test=0.886), recall=(train=0.008, test=0.006), roc_auc=(train=0.625, test=0.585), total=   9.3s\n",
      "[CV] model__max_depth=8, model__n_estimators=500, model__random_state=0 \n",
      "[CV]  model__max_depth=8, model__n_estimators=500, model__random_state=0, accuracy=(train=0.887, test=0.886), recall=(train=0.005, test=0.004), roc_auc=(train=0.622, test=0.587), total=  39.9s\n",
      "[CV] model__max_depth=8, model__n_estimators=500, model__random_state=0 \n",
      "[CV]  model__max_depth=8, model__n_estimators=500, model__random_state=0, accuracy=(train=0.886, test=0.886), recall=(train=0.007, test=0.006), roc_auc=(train=0.622, test=0.588), total=  39.2s\n",
      "[CV] model__max_depth=8, model__n_estimators=500, model__random_state=0 \n",
      "[CV]  model__max_depth=8, model__n_estimators=500, model__random_state=0, accuracy=(train=0.887, test=0.886), recall=(train=0.005, test=0.004), roc_auc=(train=0.621, test=0.584), total=  40.6s\n",
      "[CV] model__max_depth=8, model__n_estimators=500, model__random_state=0 \n",
      "[CV]  model__max_depth=8, model__n_estimators=500, model__random_state=0, accuracy=(train=0.886, test=0.887), recall=(train=0.006, test=0.006), roc_auc=(train=0.625, test=0.586), total=  40.4s\n",
      "[CV] model__max_depth=10, model__n_estimators=100, model__random_state=0 \n",
      "[CV]  model__max_depth=10, model__n_estimators=100, model__random_state=0, accuracy=(train=0.888, test=0.888), recall=(train=0.002, test=0.001), roc_auc=(train=0.672, test=0.590), total=  10.6s\n",
      "[CV] model__max_depth=10, model__n_estimators=100, model__random_state=0 \n",
      "[CV]  model__max_depth=10, model__n_estimators=100, model__random_state=0, accuracy=(train=0.888, test=0.888), recall=(train=0.002, test=0.002), roc_auc=(train=0.676, test=0.599), total=  10.5s\n",
      "[CV] model__max_depth=10, model__n_estimators=100, model__random_state=0 \n",
      "[CV]  model__max_depth=10, model__n_estimators=100, model__random_state=0, accuracy=(train=0.888, test=0.889), recall=(train=0.001, test=0.000), roc_auc=(train=0.673, test=0.595), total=  10.1s\n",
      "[CV] model__max_depth=10, model__n_estimators=100, model__random_state=0 \n",
      "[CV]  model__max_depth=10, model__n_estimators=100, model__random_state=0, accuracy=(train=0.888, test=0.888), recall=(train=0.002, test=0.001), roc_auc=(train=0.679, test=0.594), total=  10.4s\n",
      "[CV] model__max_depth=10, model__n_estimators=500, model__random_state=0 \n",
      "[CV]  model__max_depth=10, model__n_estimators=500, model__random_state=0, accuracy=(train=0.889, test=0.888), recall=(train=0.002, test=0.000), roc_auc=(train=0.677, test=0.596), total=  46.9s\n",
      "[CV] model__max_depth=10, model__n_estimators=500, model__random_state=0 \n",
      "[CV]  model__max_depth=10, model__n_estimators=500, model__random_state=0, accuracy=(train=0.888, test=0.888), recall=(train=0.001, test=0.001), roc_auc=(train=0.678, test=0.598), total=  44.3s\n",
      "[CV] model__max_depth=10, model__n_estimators=500, model__random_state=0 \n",
      "[CV]  model__max_depth=10, model__n_estimators=500, model__random_state=0, accuracy=(train=0.888, test=0.889), recall=(train=0.001, test=0.000), roc_auc=(train=0.675, test=0.594), total=  51.0s\n",
      "[CV] model__max_depth=10, model__n_estimators=500, model__random_state=0 \n",
      "[CV]  model__max_depth=10, model__n_estimators=500, model__random_state=0, accuracy=(train=0.888, test=0.888), recall=(train=0.002, test=0.000), roc_auc=(train=0.680, test=0.594), total=  47.6s\n",
      "[CV] model__max_depth=12, model__n_estimators=100, model__random_state=0 \n",
      "[CV]  model__max_depth=12, model__n_estimators=100, model__random_state=0, accuracy=(train=0.889, test=0.888), recall=(train=0.002, test=0.000), roc_auc=(train=0.760, test=0.602), total=  11.7s\n",
      "[CV] model__max_depth=12, model__n_estimators=100, model__random_state=0 \n",
      "[CV]  model__max_depth=12, model__n_estimators=100, model__random_state=0, accuracy=(train=0.889, test=0.889), recall=(train=0.003, test=0.001), roc_auc=(train=0.764, test=0.599), total=  11.6s\n",
      "[CV] model__max_depth=12, model__n_estimators=100, model__random_state=0 \n",
      "[CV]  model__max_depth=12, model__n_estimators=100, model__random_state=0, accuracy=(train=0.889, test=0.889), recall=(train=0.002, test=0.000), roc_auc=(train=0.756, test=0.604), total=  11.5s\n",
      "[CV] model__max_depth=12, model__n_estimators=100, model__random_state=0 \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  model__max_depth=12, model__n_estimators=100, model__random_state=0, accuracy=(train=0.889, test=0.888), recall=(train=0.003, test=0.000), roc_auc=(train=0.763, test=0.602), total=  11.5s\n",
      "[CV] model__max_depth=12, model__n_estimators=500, model__random_state=0 \n",
      "[CV]  model__max_depth=12, model__n_estimators=500, model__random_state=0, accuracy=(train=0.889, test=0.889), recall=(train=0.002, test=0.000), roc_auc=(train=0.764, test=0.606), total=  52.1s\n",
      "[CV] model__max_depth=12, model__n_estimators=500, model__random_state=0 \n",
      "[CV]  model__max_depth=12, model__n_estimators=500, model__random_state=0, accuracy=(train=0.889, test=0.889), recall=(train=0.001, test=0.001), roc_auc=(train=0.767, test=0.604), total=  53.1s\n",
      "[CV] model__max_depth=12, model__n_estimators=500, model__random_state=0 \n",
      "[CV]  model__max_depth=12, model__n_estimators=500, model__random_state=0, accuracy=(train=0.889, test=0.889), recall=(train=0.001, test=0.000), roc_auc=(train=0.761, test=0.605), total=  52.0s\n",
      "[CV] model__max_depth=12, model__n_estimators=500, model__random_state=0 \n",
      "[CV]  model__max_depth=12, model__n_estimators=500, model__random_state=0, accuracy=(train=0.889, test=0.888), recall=(train=0.002, test=0.000), roc_auc=(train=0.769, test=0.603), total=  54.9s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done  40 out of  40 | elapsed: 18.2min finished\n"
     ]
    }
   ],
   "source": [
    "best_model = clf.fit(X_train, y_train)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}