models_non-text.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 5a. Non-Text Features"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Import Required Modules and Load Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/johann/miniforge3/envs/d2l/lib/python3.8/site-packages/xgboost/compat.py:36: FutureWarning: pandas.Int64Index is deprecated and will be removed from pandas in a future version. Use pandas.Index with the appropriate dtype instead.\n",
      "  from pandas import MultiIndex, Int64Index\n"
     ]
    }
   ],
   "source": [
    "import pickle\n",
    "import pandas as pd\n",
    "\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.linear_model import Ridge\n",
    "from xgboost.sklearn import XGBRegressor\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "\n",
    "from catboost import CatBoostRegressor, Pool\n",
    "\n",
    "from catboost import EShapCalcType, EFeaturesSelectionAlgorithm\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.model_selection import cross_validate\n",
    "\n",
    "from keras.regularizers import l2\n",
    "from keras.layers import Dropout, Dense, Input\n",
    "from keras import Model\n",
    "from tensorflow import keras\n",
    "from keras import losses\n",
    "\n",
    "from scikeras.wrappers import KerasRegressor\n",
    "\n",
    "from keras.callbacks import EarlyStopping\n",
    "\n",
    "import model_evaluation as me\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('X_tr_1_nontext.pkl', 'rb') as file_name:\n",
    "    X_tr_1_nontext = pickle.load(file_name)\n",
    "    \n",
    "with open('X_val_1_nontext.pkl', 'rb') as file_name:\n",
    "    X_val_1_nontext = pickle.load(file_name)\n",
    "    \n",
    "with open('y_tr_1.pkl', 'rb') as file_name:\n",
    "    y_tr_1 = pickle.load(file_name)\n",
    "    \n",
    "with open('y_val_1.pkl', 'rb') as file_name:\n",
    "    y_val_1 = pickle.load(file_name)\n",
    "    \n",
    "with open('X_tr_full_nontext.pkl', 'rb') as file_name:\n",
    "    X_tr_full_nontext = pickle.load(file_name)\n",
    "    \n",
    "with open('X_tr_full_text.pkl', 'rb') as file_name:\n",
    "    X_tr_full_text = pickle.load(file_name)\n",
    "    \n",
    "with open('y_train.pkl', 'rb') as file_name:\n",
    "    y_train = pickle.load(file_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('X_test_nontext.pkl', 'rb') as file_name:\n",
    "    X_test_nontext = pickle.load(file_name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Evaluation Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "cv = KFold(n_splits=10, shuffle=True, random_state=777)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "scoring = {'negRMSE': 'neg_root_mean_squared_error', 'negMAE': 'neg_mean_absolute_error', 'R_squared': 'r2'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_name = []\n",
    "cv_RMSE = []\n",
    "cv_MAE = []\n",
    "cv_R2 = []"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 1. Ridge Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "ridge = Ridge()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "score = cross_validate(ridge, X_tr_full_nontext, y_train, scoring=scoring, cv=cv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_name.append('Ridge Regression')\n",
    "cv_RMSE.append(-score['test_negRMSE'].mean())\n",
    "cv_MAE.append(-score['test_negMAE'].mean())\n",
    "cv_R2.append(score['test_R_squared'].mean())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2. Random Forest Regressor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=2, estimator=RandomForestRegressor(),\n",
       "             param_grid={'max_features': ['sqrt'], 'min_samples_leaf': [3, 5],\n",
       "                         'min_samples_split': [3, 4, 5], 'n_estimators': [750],\n",
       "                         'random_state': [123]},\n",
       "             scoring='neg_root_mean_squared_error', verbose=False)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rf = RandomForestRegressor()\n",
    "\n",
    "parameters = {\"n_estimators\" : [750],\n",
    "             \"max_features\" : [\"sqrt\"],\n",
    "             \"min_samples_split\" : [3, 4, 5],\n",
    "             \"min_samples_leaf\" : [3, 5],\n",
    "             \"random_state\" : [123]\n",
    "             }\n",
    "\n",
    "rf_grid = GridSearchCV(rf,\n",
    "                        parameters,\n",
    "                        cv = 2,\n",
    "                        scoring='neg_root_mean_squared_error',\n",
    "                        verbose=False)\n",
    "\n",
    "rf_grid.fit(X_tr_full_nontext, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_params = rf_grid.best_params_\n",
    "\n",
    "rf_reg = RandomForestRegressor(**best_params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "score = cross_validate(rf_reg, X_tr_full_nontext, y_train, scoring=scoring, cv=cv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_name.append('Random Forest Regressor')\n",
    "cv_RMSE.append(-score['test_negRMSE'].mean())\n",
    "cv_MAE.append(-score['test_negMAE'].mean())\n",
    "cv_R2.append(score['test_R_squared'].mean())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 3. XGB Regressor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=2,\n",
       "             estimator=XGBRegressor(base_score=None, booster=None,\n",
       "                                    colsample_bylevel=None,\n",
       "                                    colsample_bynode=None,\n",
       "                                    colsample_bytree=None,\n",
       "                                    enable_categorical=False, gamma=None,\n",
       "                                    gpu_id=None, importance_type=None,\n",
       "                                    interaction_constraints=None,\n",
       "                                    learning_rate=None, max_delta_step=None,\n",
       "                                    max_depth=None, min_child_weight=None,\n",
       "                                    missing=nan, monotone_constraints=None,\n",
       "                                    n...\n",
       "                                    random_state=None, reg_alpha=None,\n",
       "                                    reg_lambda=None, scale_pos_weight=None,\n",
       "                                    subsample=None, tree_method=None,\n",
       "                                    validate_parameters=None, verbosity=None),\n",
       "             n_jobs=-1,\n",
       "             param_grid={'colsample_bytree': [0.7],\n",
       "                         'learning_rate': [0.03, 0.04, 0.05], 'max_depth': [8],\n",
       "                         'min_child_weight': [4, 6], 'n_estimators': [800],\n",
       "                         'random_state': [123]},\n",
       "             scoring='neg_root_mean_squared_error', verbose=False)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb_reg = XGBRegressor()\n",
    "\n",
    "parameters = {'learning_rate': [0.03, 0.04, 0.05],\n",
    "              'max_depth': [8],\n",
    "              'min_child_weight': [4, 6],\n",
    "              'colsample_bytree': [0.7],\n",
    "              'n_estimators': [800],\n",
    "              'random_state': [123]}\n",
    "\n",
    "xgb_grid = GridSearchCV(xgb_reg,\n",
    "                        parameters,\n",
    "                        cv = 2,\n",
    "                        n_jobs = -1,\n",
    "                        scoring='neg_root_mean_squared_error',\n",
    "                        verbose=False)\n",
    "\n",
    "xgb_grid.fit(X_tr_full_nontext, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "xgb_reg = XGBRegressor(**xgb_grid.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "score = cross_validate(xgb_reg, X_tr_full_nontext, y_train, scoring=scoring, cv=cv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_name.append('XGB Regressor')\n",
    "cv_RMSE.append(-score['test_negRMSE'].mean())\n",
    "cv_MAE.append(-score['test_negMAE'].mean())\n",
    "cv_R2.append(score['test_R_squared'].mean())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 4. CatBoost Regressor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=2,\n",
       "             estimator=<catboost.core.CatBoostRegressor object at 0x17116f850>,\n",
       "             n_jobs=-1,\n",
       "             param_grid={'depth': [6, 8, 10], 'early_stopping_rounds': [20],\n",
       "                         'iterations': [2000], 'l2_leaf_reg': [2, 3, 4],\n",
       "                         'learning_rate': [0.02, 0.03, 0.04], 'verbose': [0]},\n",
       "             scoring='neg_root_mean_squared_error', verbose=False)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "catboost_params = {\n",
    "    'iterations': [2000],\n",
    "    'depth': [6, 8, 10],\n",
    "    'learning_rate': [0.02, 0.03, 0.04],\n",
    "    'l2_leaf_reg': [2, 3, 4],\n",
    "    'early_stopping_rounds': [20],\n",
    "    'verbose': [0]\n",
    "}\n",
    "\n",
    "\n",
    "cbr = CatBoostRegressor()\n",
    "\n",
    "cbr_grid = GridSearchCV(cbr,\n",
    "                        catboost_params,\n",
    "                        cv = 2,\n",
    "                        n_jobs = -1,\n",
    "                        scoring='neg_root_mean_squared_error',\n",
    "                        verbose=False)\n",
    "\n",
    "cbr_grid.fit(X_tr_full_nontext, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_params = cbr_grid.best_params_\n",
    "best_params['iterations'] = 1500\n",
    "best_params['verbose'] = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "cbr_reg = CatBoostRegressor(**best_params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "score = cross_validate(cbr_reg, X_tr_full_nontext, y_train, scoring=scoring, cv=cv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_name.append('CatBoostRegressor')\n",
    "cv_RMSE.append(-score['test_negRMSE'].mean())\n",
    "cv_MAE.append(-score['test_negMAE'].mean())\n",
    "cv_R2.append(score['test_R_squared'].mean())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 5. Feature Selection from CatBoost Regressor "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_pool = Pool(X_tr_1_nontext, y_tr_1)\n",
    "test_pool = Pool(X_val_1_nontext, y_val_1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "summary = cbr_reg.select_features(\n",
    "    train_pool,\n",
    "    eval_set=test_pool,\n",
    "    features_for_select=['host_response_rate-flag_missing_description'],\n",
    "    num_features_to_select=70,\n",
    "    steps=3,\n",
    "    algorithm=EFeaturesSelectionAlgorithm.RecursiveByShapValues,\n",
    "    shap_calc_type=EShapCalcType.Regular,\n",
    "    train_final_model=True,\n",
    "    logging_level='Silent',\n",
    "    plot=False\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_tr_1_nontext_sel = X_tr_1_nontext.iloc[:,summary['selected_features']]\n",
    "X_val_1_nontext_sel = X_val_1_nontext.iloc[:,summary['selected_features']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_tr_full_nontext_sel = X_tr_full_nontext.iloc[:,summary['selected_features']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_test_nontext_sel = X_test_nontext.iloc[:,summary['selected_features']]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We save the pruned datasets."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('X_tr_1_nontext_sel.pkl', 'wb') as file_name:\n",
    "    pickle.dump(X_tr_1_nontext_sel, file_name)\n",
    "    \n",
    "with open('X_val_1_nontext_sel.pkl', 'wb') as file_name:\n",
    "    pickle.dump(X_val_1_nontext_sel, file_name)\n",
    "    \n",
    "with open('X_tr_full_nontext_sel.pkl', 'wb') as file_name:\n",
    "    pickle.dump(X_tr_full_nontext_sel, file_name)\n",
    "    \n",
    "with open('X_test_nontext_sel.pkl', 'wb') as file_name:\n",
    "    pickle.dump(X_test_nontext_sel, file_name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 5.1 XGB with selected features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "score = cross_validate(xgb_reg, X_tr_full_nontext_sel, y_train, scoring=scoring, cv=cv)\n",
    "\n",
    "model_name.append('XGB Regressor (Feat. Sel.)')\n",
    "cv_RMSE.append(-score['test_negRMSE'].mean())\n",
    "cv_MAE.append(-score['test_negMAE'].mean())\n",
    "cv_R2.append(score['test_R_squared'].mean())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 5.2 CBR with selected features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=2,\n",
       "             estimator=<catboost.core.CatBoostRegressor object at 0x17ce7ab20>,\n",
       "             n_jobs=-1,\n",
       "             param_grid={'depth': [10], 'early_stopping_rounds': [20],\n",
       "                         'iterations': [2000], 'l2_leaf_reg': [3, 4, 5],\n",
       "                         'learning_rate': [0.03, 0.035, 0.04], 'verbose': [0]},\n",
       "             scoring='neg_root_mean_squared_error', verbose=False)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "catboost_params = {\n",
    "    'iterations': [2000],\n",
    "    'depth': [10],\n",
    "    'learning_rate': [0.03, 0.035, 0.04],\n",
    "    'l2_leaf_reg': [3, 4, 5],\n",
    "    'early_stopping_rounds': [20],\n",
    "    'verbose': [0]\n",
    "}\n",
    "\n",
    "\n",
    "cbr = CatBoostRegressor()\n",
    "\n",
    "cbr_grid = GridSearchCV(cbr,\n",
    "                        catboost_params,\n",
    "                        cv = 2,\n",
    "                        n_jobs = -1,\n",
    "                        scoring='neg_root_mean_squared_error',\n",
    "                        verbose=False)\n",
    "\n",
    "cbr_grid.fit(X_tr_1_nontext_sel, y_tr_1, eval_set=(X_val_1_nontext_sel, y_val_1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "cbr_reg = CatBoostRegressor(**cbr_grid.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "score = cross_validate(cbr_reg, X_tr_full_nontext_sel, y_train, scoring=scoring, cv=cv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_name.append('CatBoostRegressor (Feat. Sel.)')\n",
    "cv_RMSE.append(-score['test_negRMSE'].mean())\n",
    "cv_MAE.append(-score['test_negMAE'].mean())\n",
    "cv_R2.append(score['test_R_squared'].mean())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 5.3 Neural Network with Dropout Regularization\n",
    "Based on [this source](https://www.adriangb.com/scikeras/stable/notebooks/Basic_Usage.html#3.-Training-a-regressor)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "callbacks = [EarlyStopping(monitor=\"val_loss\", patience=3, verbose=1, min_delta=0.1, restore_best_weights=True)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_reg_nn(n_hidden=[256, 128, 64], dropout=0.025, reg=0.025, N=X_tr_1_nontext_sel.shape[1]):\n",
    "    \n",
    "    n_epochs = 15\n",
    "    batch_size = 128\n",
    "    \n",
    "    inputs = Input(shape=(N,))\n",
    "    inter = Dropout(dropout)(inputs)\n",
    "    inter = Dense(n_hidden[0], activation='relu', kernel_initializer=keras.initializers.he_normal(seed=123), kernel_regularizer=l2(reg))(inter)\n",
    "    for i in range(len(n_hidden) - 1):\n",
    "        inter = Dropout(dropout)(inter)\n",
    "        inter = Dense(n_hidden[i+1], activation='relu', kernel_initializer=keras.initializers.he_normal(seed=123), kernel_regularizer=l2(reg))(inter)\n",
    "    inter = Dropout(dropout)(inter)\n",
    "    outputs = Dense(1)(inter)\n",
    "    model = Model(inputs, outputs)\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "reg = KerasRegressor(\n",
    "    model=get_reg_nn(N=X_tr_full_nontext_sel.shape[1]),\n",
    "    optimizer='adam',\n",
    "    loss='mean_squared_error',\n",
    "    metrics=[me.root_mean_squared_error],\n",
    "    verbose=0\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Compiled the loaded model, but the compiled metrics have yet to be built. `model.compile_metrics` will be empty until you train or evaluate the model.\n",
      "INFO:tensorflow:Assets written to: ram:///var/folders/ry/rzz66hz547v4ph_l8p5sc_d00000gn/T/tmpdicw7pvp/assets\n",
      "WARNING:tensorflow:No training configuration found in save file, so the model was *not* compiled. Compile it manually.\n",
      "WARNING:tensorflow:Compiled the loaded model, but the compiled metrics have yet to be built. `model.compile_metrics` will be empty until you train or evaluate the model.\n",
      "INFO:tensorflow:Assets written to: ram:///var/folders/ry/rzz66hz547v4ph_l8p5sc_d00000gn/T/tmpqz85kkvx/assets\n",
      "WARNING:tensorflow:No training configuration found in save file, so the model was *not* compiled. Compile it manually.\n",
      "WARNING:tensorflow:Compiled the loaded model, but the compiled metrics have yet to be built. `model.compile_metrics` will be empty until you train or evaluate the model.\n",
      "INFO:tensorflow:Assets written to: ram:///var/folders/ry/rzz66hz547v4ph_l8p5sc_d00000gn/T/tmpubv_1_rz/assets\n",
      "WARNING:tensorflow:No training configuration found in save file, so the model was *not* compiled. Compile it manually.\n",
      "WARNING:tensorflow:Compiled the loaded model, but the compiled metrics have yet to be built. `model.compile_metrics` will be empty until you train or evaluate the model.\n",
      "INFO:tensorflow:Assets written to: ram:///var/folders/ry/rzz66hz547v4ph_l8p5sc_d00000gn/T/tmpb9dkhm83/assets\n",
      "WARNING:tensorflow:No training configuration found in save file, so the model was *not* compiled. Compile it manually.\n",
      "WARNING:tensorflow:Compiled the loaded model, but the compiled metrics have yet to be built. `model.compile_metrics` will be empty until you train or evaluate the model.\n",
      "INFO:tensorflow:Assets written to: ram:///var/folders/ry/rzz66hz547v4ph_l8p5sc_d00000gn/T/tmp6ctlpsye/assets\n",
      "WARNING:tensorflow:No training configuration found in save file, so the model was *not* compiled. Compile it manually.\n",
      "WARNING:tensorflow:Compiled the loaded model, but the compiled metrics have yet to be built. `model.compile_metrics` will be empty until you train or evaluate the model.\n",
      "INFO:tensorflow:Assets written to: ram:///var/folders/ry/rzz66hz547v4ph_l8p5sc_d00000gn/T/tmpsb456dh2/assets\n",
      "WARNING:tensorflow:No training configuration found in save file, so the model was *not* compiled. Compile it manually.\n",
      "WARNING:tensorflow:Compiled the loaded model, but the compiled metrics have yet to be built. `model.compile_metrics` will be empty until you train or evaluate the model.\n",
      "INFO:tensorflow:Assets written to: ram:///var/folders/ry/rzz66hz547v4ph_l8p5sc_d00000gn/T/tmp3liifmxg/assets\n",
      "WARNING:tensorflow:No training configuration found in save file, so the model was *not* compiled. Compile it manually.\n",
      "WARNING:tensorflow:Compiled the loaded model, but the compiled metrics have yet to be built. `model.compile_metrics` will be empty until you train or evaluate the model.\n",
      "INFO:tensorflow:Assets written to: ram:///var/folders/ry/rzz66hz547v4ph_l8p5sc_d00000gn/T/tmpspipppf3/assets\n",
      "WARNING:tensorflow:No training configuration found in save file, so the model was *not* compiled. Compile it manually.\n",
      "WARNING:tensorflow:Compiled the loaded model, but the compiled metrics have yet to be built. `model.compile_metrics` will be empty until you train or evaluate the model.\n",
      "INFO:tensorflow:Assets written to: ram:///var/folders/ry/rzz66hz547v4ph_l8p5sc_d00000gn/T/tmpkh39xavt/assets\n",
      "WARNING:tensorflow:No training configuration found in save file, so the model was *not* compiled. Compile it manually.\n",
      "WARNING:tensorflow:Compiled the loaded model, but the compiled metrics have yet to be built. `model.compile_metrics` will be empty until you train or evaluate the model.\n",
      "INFO:tensorflow:Assets written to: ram:///var/folders/ry/rzz66hz547v4ph_l8p5sc_d00000gn/T/tmpqqnblaat/assets\n",
      "WARNING:tensorflow:No training configuration found in save file, so the model was *not* compiled. Compile it manually.\n"
     ]
    }
   ],
   "source": [
    "score = cross_validate(reg, X_tr_full_nontext_sel, y_train, scoring=scoring, cv=cv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_name.append('Deep Neural Network (Feat. Sel.)')\n",
    "cv_RMSE.append(-score['test_negRMSE'].mean())\n",
    "cv_MAE.append(-score['test_negMAE'].mean())\n",
    "cv_R2.append(score['test_R_squared'].mean())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Export Results Table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "results = pd.DataFrame({'Model Name': model_name, 'RMSE': cv_RMSE, 'MAE': cv_MAE, 'R-squared': cv_R2})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "results.to_csv('results_models_nontext.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Model Name</th>\n",
       "      <th>RMSE</th>\n",
       "      <th>MAE</th>\n",
       "      <th>R-squared</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Ridge Regression</td>\n",
       "      <td>54.554011</td>\n",
       "      <td>34.767277</td>\n",
       "      <td>0.575427</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Random Forest Regressor</td>\n",
       "      <td>48.996181</td>\n",
       "      <td>29.725004</td>\n",
       "      <td>0.657523</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>XGB Regressor</td>\n",
       "      <td>45.370005</td>\n",
       "      <td>27.335242</td>\n",
       "      <td>0.706233</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>CatBoostRegressor</td>\n",
       "      <td>45.562294</td>\n",
       "      <td>27.495034</td>\n",
       "      <td>0.703754</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>XGB Regressor (Feat. Sel.)</td>\n",
       "      <td>45.457703</td>\n",
       "      <td>27.379079</td>\n",
       "      <td>0.705108</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>CatBoostRegressor (Feat. Sel.)</td>\n",
       "      <td>45.074265</td>\n",
       "      <td>27.108501</td>\n",
       "      <td>0.710033</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Deep Neural Network (Feat. Sel.)</td>\n",
       "      <td>51.983582</td>\n",
       "      <td>31.575575</td>\n",
       "      <td>0.614442</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                         Model Name       RMSE        MAE  R-squared\n",
       "0                  Ridge Regression  54.554011  34.767277   0.575427\n",
       "1           Random Forest Regressor  48.996181  29.725004   0.657523\n",
       "2                     XGB Regressor  45.370005  27.335242   0.706233\n",
       "3                 CatBoostRegressor  45.562294  27.495034   0.703754\n",
       "4        XGB Regressor (Feat. Sel.)  45.457703  27.379079   0.705108\n",
       "5    CatBoostRegressor (Feat. Sel.)  45.074265  27.108501   0.710033\n",
       "6  Deep Neural Network (Feat. Sel.)  51.983582  31.575575   0.614442"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}