notebooks/dataprocessing/module_tests/iterative_imputation.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# IterativeDataImputer Example\n",
    "\n",
    "In this example, we will explore how to perform imputation with the HR promotion dataset using iterative imputation. \n",
    "\n",
    "This method imputes missing data of a feature using the other features. It uses a round-robin method of modeling each feature with missing values to be imputed as a function of the other features. \n",
    "\n",
    "This subclass uses the class `sklearn.impute.IterativeImputer` class from `sklearn` in the background (note that this sklearn class is still in an experimental stage)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append('../../../notebooks')\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from raimitigations.dataprocessing import IterativeDataImputer\n",
    "\n",
    "from download import download_datasets\n",
    "\n",
    "from sklearn.linear_model import BayesianRidge, Ridge\n",
    "from sklearn.ensemble import RandomForestRegressor"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Handling a DataFrame with column names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>employee_id</th>\n",
       "      <th>department</th>\n",
       "      <th>region</th>\n",
       "      <th>education</th>\n",
       "      <th>gender</th>\n",
       "      <th>recruitment_channel</th>\n",
       "      <th>no_of_trainings</th>\n",
       "      <th>age</th>\n",
       "      <th>previous_year_rating</th>\n",
       "      <th>length_of_service</th>\n",
       "      <th>KPIs_met &gt;80%</th>\n",
       "      <th>awards_won?</th>\n",
       "      <th>avg_training_score</th>\n",
       "      <th>is_promoted</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>65438</td>\n",
       "      <td>Sales &amp; Marketing</td>\n",
       "      <td>region_7</td>\n",
       "      <td>Master's &amp; above</td>\n",
       "      <td>f</td>\n",
       "      <td>sourcing</td>\n",
       "      <td>1</td>\n",
       "      <td>35</td>\n",
       "      <td>5.0</td>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>49</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>65141</td>\n",
       "      <td>Operations</td>\n",
       "      <td>region_22</td>\n",
       "      <td>Bachelor's</td>\n",
       "      <td>m</td>\n",
       "      <td>other</td>\n",
       "      <td>1</td>\n",
       "      <td>30</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>60</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7513</td>\n",
       "      <td>Sales &amp; Marketing</td>\n",
       "      <td>region_19</td>\n",
       "      <td>Bachelor's</td>\n",
       "      <td>m</td>\n",
       "      <td>sourcing</td>\n",
       "      <td>1</td>\n",
       "      <td>34</td>\n",
       "      <td>3.0</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>50</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2542</td>\n",
       "      <td>Sales &amp; Marketing</td>\n",
       "      <td>region_23</td>\n",
       "      <td>Bachelor's</td>\n",
       "      <td>m</td>\n",
       "      <td>other</td>\n",
       "      <td>2</td>\n",
       "      <td>39</td>\n",
       "      <td>1.0</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>50</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>48945</td>\n",
       "      <td>Technology</td>\n",
       "      <td>region_26</td>\n",
       "      <td>Bachelor's</td>\n",
       "      <td>m</td>\n",
       "      <td>other</td>\n",
       "      <td>1</td>\n",
       "      <td>45</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>73</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9995</th>\n",
       "      <td>14934</td>\n",
       "      <td>Procurement</td>\n",
       "      <td>region_13</td>\n",
       "      <td>Master's &amp; above</td>\n",
       "      <td>f</td>\n",
       "      <td>other</td>\n",
       "      <td>1</td>\n",
       "      <td>37</td>\n",
       "      <td>4.0</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>71</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9996</th>\n",
       "      <td>22040</td>\n",
       "      <td>Sales &amp; Marketing</td>\n",
       "      <td>region_33</td>\n",
       "      <td>Master's &amp; above</td>\n",
       "      <td>m</td>\n",
       "      <td>sourcing</td>\n",
       "      <td>1</td>\n",
       "      <td>39</td>\n",
       "      <td>3.0</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>48</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9997</th>\n",
       "      <td>14188</td>\n",
       "      <td>Finance</td>\n",
       "      <td>region_13</td>\n",
       "      <td>Master's &amp; above</td>\n",
       "      <td>f</td>\n",
       "      <td>sourcing</td>\n",
       "      <td>1</td>\n",
       "      <td>33</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>58</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9998</th>\n",
       "      <td>73566</td>\n",
       "      <td>Operations</td>\n",
       "      <td>region_28</td>\n",
       "      <td>Master's &amp; above</td>\n",
       "      <td>m</td>\n",
       "      <td>other</td>\n",
       "      <td>1</td>\n",
       "      <td>32</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>57</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9999</th>\n",
       "      <td>21372</td>\n",
       "      <td>Procurement</td>\n",
       "      <td>region_13</td>\n",
       "      <td>Bachelor's</td>\n",
       "      <td>f</td>\n",
       "      <td>sourcing</td>\n",
       "      <td>1</td>\n",
       "      <td>32</td>\n",
       "      <td>3.0</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>71</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10000 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      employee_id         department     region         education gender  \\\n",
       "0           65438  Sales & Marketing   region_7  Master's & above      f   \n",
       "1           65141         Operations  region_22        Bachelor's      m   \n",
       "2            7513  Sales & Marketing  region_19        Bachelor's      m   \n",
       "3            2542  Sales & Marketing  region_23        Bachelor's      m   \n",
       "4           48945         Technology  region_26        Bachelor's      m   \n",
       "...           ...                ...        ...               ...    ...   \n",
       "9995        14934        Procurement  region_13  Master's & above      f   \n",
       "9996        22040  Sales & Marketing  region_33  Master's & above      m   \n",
       "9997        14188            Finance  region_13  Master's & above      f   \n",
       "9998        73566         Operations  region_28  Master's & above      m   \n",
       "9999        21372        Procurement  region_13        Bachelor's      f   \n",
       "\n",
       "     recruitment_channel  no_of_trainings  age  previous_year_rating  \\\n",
       "0               sourcing                1   35                   5.0   \n",
       "1                  other                1   30                   5.0   \n",
       "2               sourcing                1   34                   3.0   \n",
       "3                  other                2   39                   1.0   \n",
       "4                  other                1   45                   3.0   \n",
       "...                  ...              ...  ...                   ...   \n",
       "9995               other                1   37                   4.0   \n",
       "9996            sourcing                1   39                   3.0   \n",
       "9997            sourcing                1   33                   4.0   \n",
       "9998               other                1   32                   4.0   \n",
       "9999            sourcing                1   32                   3.0   \n",
       "\n",
       "      length_of_service  KPIs_met >80%  awards_won?  avg_training_score  \\\n",
       "0                     8              1            0                  49   \n",
       "1                     4              0            0                  60   \n",
       "2                     7              0            0                  50   \n",
       "3                    10              0            0                  50   \n",
       "4                     2              0            0                  73   \n",
       "...                 ...            ...          ...                 ...   \n",
       "9995                  7              1            0                  71   \n",
       "9996                  7              0            0                  48   \n",
       "9997                  4              1            0                  58   \n",
       "9998                  4              1            0                  57   \n",
       "9999                  6              0            0                  71   \n",
       "\n",
       "      is_promoted  \n",
       "0               0  \n",
       "1               0  \n",
       "2               0  \n",
       "3               0  \n",
       "4               0  \n",
       "...           ...  \n",
       "9995            0  \n",
       "9996            0  \n",
       "9997            0  \n",
       "9998            1  \n",
       "9999            0  \n",
       "\n",
       "[10000 rows x 14 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_dir = '../../../datasets/'\n",
    "download_datasets(data_dir)\n",
    "dataset = pd.read_csv(data_dir + 'hr_promotion/train.csv')\n",
    "dataset = dataset[:10000]\n",
    "\n",
    "dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "employee_id             False\n",
      "department              False\n",
      "region                  False\n",
      "education                True\n",
      "gender                  False\n",
      "recruitment_channel     False\n",
      "no_of_trainings         False\n",
      "age                     False\n",
      "previous_year_rating     True\n",
      "length_of_service       False\n",
      "KPIs_met >80%           False\n",
      "awards_won?             False\n",
      "avg_training_score      False\n",
      "is_promoted             False\n",
      "dtype: bool\n",
      "[\"Master's & above\" \"Bachelor's\" nan 'Below Secondary']\n",
      "[ 5.  3.  1.  4. nan  2.]\n"
     ]
    }
   ],
   "source": [
    "print(dataset.isna().any())\n",
    "print(dataset['education'].unique())\n",
    "print(dataset['previous_year_rating'].unique())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Without Enabling Encoding"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As we can see above, both the **education** and **previous_year_rating** have missing values. \n",
    "\n",
    "However, note that the dataset includes categorical columns such as **education**, while sklearn's `sklearn.IterativeImputer` can only handle numerical data. The good thing is, `IteratveDataImputer` can in fact handle categorical data using the boolean parameter `enable_encoder`. \n",
    "\n",
    "First, let's try to use the default value `enable_encoder=False`, categorical data will be excluded from the imputation process, whether it has missing values or not. We can also use `col_impute` to specify only `previous_year_rating` to be imputed.\n",
    "\n",
    "Additionally, we can specify `sklearn.IterativeImputer`'s parameters using the `iterative_params` dictionary as long as it uses the following format (if you choose not to pass anything to this param, default values will be used)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "WARNING: Categorical columns will be excluded from the iterative imputation process.\n",
      "If you'd like to include these columns, you need to set 'enable_encoder'=True.\n",
      "If you'd like to use a different type of encoding before imputation, consider using the Pipeline class and call your own encoder before calling this subclass for imputation.\n",
      "[IterativeImputer] Completing matrix with shape (10000, 9)\n",
      "[IterativeImputer] Change: 1.5071906463137383, scaled tolerance: 78.297 \n",
      "[IterativeImputer] Early stopping criterion reached.\n",
      "[IterativeImputer] Completing matrix with shape (10000, 9)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>employee_id</th>\n",
       "      <th>no_of_trainings</th>\n",
       "      <th>age</th>\n",
       "      <th>previous_year_rating</th>\n",
       "      <th>length_of_service</th>\n",
       "      <th>KPIs_met &gt;80%</th>\n",
       "      <th>awards_won?</th>\n",
       "      <th>avg_training_score</th>\n",
       "      <th>is_promoted</th>\n",
       "      <th>gender</th>\n",
       "      <th>recruitment_channel</th>\n",
       "      <th>education</th>\n",
       "      <th>region</th>\n",
       "      <th>department</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>65438.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>35.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>49.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>f</td>\n",
       "      <td>sourcing</td>\n",
       "      <td>Master's &amp; above</td>\n",
       "      <td>region_7</td>\n",
       "      <td>Sales &amp; Marketing</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>65141.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>30.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>60.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>m</td>\n",
       "      <td>other</td>\n",
       "      <td>Bachelor's</td>\n",
       "      <td>region_22</td>\n",
       "      <td>Operations</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7513.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>34.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>50.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>m</td>\n",
       "      <td>sourcing</td>\n",
       "      <td>Bachelor's</td>\n",
       "      <td>region_19</td>\n",
       "      <td>Sales &amp; Marketing</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2542.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>39.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>50.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>m</td>\n",
       "      <td>other</td>\n",
       "      <td>Bachelor's</td>\n",
       "      <td>region_23</td>\n",
       "      <td>Sales &amp; Marketing</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>48945.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>45.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>73.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>m</td>\n",
       "      <td>other</td>\n",
       "      <td>Bachelor's</td>\n",
       "      <td>region_26</td>\n",
       "      <td>Technology</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9995</th>\n",
       "      <td>14934.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>37.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>71.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>f</td>\n",
       "      <td>other</td>\n",
       "      <td>Master's &amp; above</td>\n",
       "      <td>region_13</td>\n",
       "      <td>Procurement</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9996</th>\n",
       "      <td>22040.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>39.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>48.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>m</td>\n",
       "      <td>sourcing</td>\n",
       "      <td>Master's &amp; above</td>\n",
       "      <td>region_33</td>\n",
       "      <td>Sales &amp; Marketing</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9997</th>\n",
       "      <td>14188.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>33.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>58.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>f</td>\n",
       "      <td>sourcing</td>\n",
       "      <td>Master's &amp; above</td>\n",
       "      <td>region_13</td>\n",
       "      <td>Finance</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9998</th>\n",
       "      <td>73566.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>32.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>57.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>m</td>\n",
       "      <td>other</td>\n",
       "      <td>Master's &amp; above</td>\n",
       "      <td>region_28</td>\n",
       "      <td>Operations</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9999</th>\n",
       "      <td>21372.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>32.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>71.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>f</td>\n",
       "      <td>sourcing</td>\n",
       "      <td>Bachelor's</td>\n",
       "      <td>region_13</td>\n",
       "      <td>Procurement</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10000 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      employee_id  no_of_trainings   age  previous_year_rating  \\\n",
       "0         65438.0              1.0  35.0                   5.0   \n",
       "1         65141.0              1.0  30.0                   5.0   \n",
       "2          7513.0              1.0  34.0                   3.0   \n",
       "3          2542.0              2.0  39.0                   1.0   \n",
       "4         48945.0              1.0  45.0                   3.0   \n",
       "...           ...              ...   ...                   ...   \n",
       "9995      14934.0              1.0  37.0                   4.0   \n",
       "9996      22040.0              1.0  39.0                   3.0   \n",
       "9997      14188.0              1.0  33.0                   4.0   \n",
       "9998      73566.0              1.0  32.0                   4.0   \n",
       "9999      21372.0              1.0  32.0                   3.0   \n",
       "\n",
       "      length_of_service  KPIs_met >80%  awards_won?  avg_training_score  \\\n",
       "0                   8.0            1.0          0.0                49.0   \n",
       "1                   4.0            0.0          0.0                60.0   \n",
       "2                   7.0            0.0          0.0                50.0   \n",
       "3                  10.0            0.0          0.0                50.0   \n",
       "4                   2.0            0.0          0.0                73.0   \n",
       "...                 ...            ...          ...                 ...   \n",
       "9995                7.0            1.0          0.0                71.0   \n",
       "9996                7.0            0.0          0.0                48.0   \n",
       "9997                4.0            1.0          0.0                58.0   \n",
       "9998                4.0            1.0          0.0                57.0   \n",
       "9999                6.0            0.0          0.0                71.0   \n",
       "\n",
       "      is_promoted gender recruitment_channel         education     region  \\\n",
       "0             0.0      f            sourcing  Master's & above   region_7   \n",
       "1             0.0      m               other        Bachelor's  region_22   \n",
       "2             0.0      m            sourcing        Bachelor's  region_19   \n",
       "3             0.0      m               other        Bachelor's  region_23   \n",
       "4             0.0      m               other        Bachelor's  region_26   \n",
       "...           ...    ...                 ...               ...        ...   \n",
       "9995          0.0      f               other  Master's & above  region_13   \n",
       "9996          0.0      m            sourcing  Master's & above  region_33   \n",
       "9997          0.0      f            sourcing  Master's & above  region_13   \n",
       "9998          1.0      m               other  Master's & above  region_28   \n",
       "9999          0.0      f            sourcing        Bachelor's  region_13   \n",
       "\n",
       "             department  \n",
       "0     Sales & Marketing  \n",
       "1            Operations  \n",
       "2     Sales & Marketing  \n",
       "3     Sales & Marketing  \n",
       "4            Technology  \n",
       "...                 ...  \n",
       "9995        Procurement  \n",
       "9996  Sales & Marketing  \n",
       "9997            Finance  \n",
       "9998         Operations  \n",
       "9999        Procurement  \n",
       "\n",
       "[10000 rows x 14 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "imputer = IterativeDataImputer(\n",
    "    df=dataset,\n",
    "    col_impute=['previous_year_rating'],\n",
    "    enable_encoder=False,\n",
    "    iterative_params={\n",
    "        'estimator': RandomForestRegressor(),\n",
    "        'missing_values': np.nan,\n",
    "        'sample_posterior': False,\n",
    "        'max_iter': 10,\n",
    "        'tol': 1e-3,\n",
    "        'n_nearest_features': None,\n",
    "        'initial_strategy': 'mean',\n",
    "        'imputation_order': 'ascending',\n",
    "        'skip_complete': False,\n",
    "        'min_value': -np.inf,\n",
    "        'max_value': np.inf,\n",
    "        'random_state': None}\n",
    ")\n",
    "imputer.fit()\n",
    "new_df = imputer.transform(dataset)\n",
    "new_df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "employee_id             False\n",
      "no_of_trainings         False\n",
      "age                     False\n",
      "previous_year_rating    False\n",
      "length_of_service       False\n",
      "KPIs_met >80%           False\n",
      "awards_won?             False\n",
      "avg_training_score      False\n",
      "is_promoted             False\n",
      "gender                  False\n",
      "recruitment_channel     False\n",
      "education                True\n",
      "region                  False\n",
      "department              False\n",
      "dtype: bool\n"
     ]
    }
   ],
   "source": [
    "print(new_df.isna().any())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### With Enabling Encoding"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now if we use `enable_encoder=True`, we can include categorical data both to be imputed and to be used in the imputation of other features.\n",
    "\n",
    "Using the original dataset, we know that both **education** and **previous_year_rating** have missing values. Below, we will not specify the columns to use for imputation (`IterativeDataImputer` will determine this for us)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> The `enable_encoder` parameter allows the class to use ordinal encoding internally before imputation, however, it uses the default 'auto' option for its `categories` parameter. We recommend that you use this internal encoding function when you aren't looking to specify ordinal semantics or order. Otherwise, you should consider using the Pipeline class to apply custom encoding before calling the `IterativeDataImputer` class for imputation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "WARNING: 'enable_encoder'=True and categorical columns will be encoded using ordinal encoding before applying the iterative imputation process.\n",
      "If you'd like to use a different type of encoding before imputation, consider using the Pipeline class and call your own encoder before calling this subclass for imputation.\n",
      "[IterativeImputer] Completing matrix with shape (10000, 14)\n",
      "[IterativeImputer] Change: 1.624667046565469, scaled tolerance: 78.297 \n",
      "[IterativeImputer] Early stopping criterion reached.\n",
      "No columns specified for imputation. These columns have been automatically identified at transform time:\n",
      "['education', 'previous_year_rating']\n",
      "\n",
      "WARNING: Note that encoded columns are not guaranteed to reverse transform if they have imputed values.\n",
      "If you'd like to use a different type of encoding before imputation, consider using the Pipeline class and call your own encoder before calling this subclass.\n",
      "[IterativeImputer] Completing matrix with shape (10000, 14)\n",
      "\n",
      "Imputed categorical columns' reverse encoding transformation complete.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>employee_id</th>\n",
       "      <th>department</th>\n",
       "      <th>region</th>\n",
       "      <th>education</th>\n",
       "      <th>gender</th>\n",
       "      <th>recruitment_channel</th>\n",
       "      <th>no_of_trainings</th>\n",
       "      <th>age</th>\n",
       "      <th>previous_year_rating</th>\n",
       "      <th>length_of_service</th>\n",
       "      <th>KPIs_met &gt;80%</th>\n",
       "      <th>awards_won?</th>\n",
       "      <th>avg_training_score</th>\n",
       "      <th>is_promoted</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>65438.0</td>\n",
       "      <td>Sales &amp; Marketing</td>\n",
       "      <td>region_7</td>\n",
       "      <td>Master's &amp; above</td>\n",
       "      <td>f</td>\n",
       "      <td>sourcing</td>\n",
       "      <td>1.0</td>\n",
       "      <td>35.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>49.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>65141.0</td>\n",
       "      <td>Operations</td>\n",
       "      <td>region_22</td>\n",
       "      <td>Bachelor's</td>\n",
       "      <td>m</td>\n",
       "      <td>other</td>\n",
       "      <td>1.0</td>\n",
       "      <td>30.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>60.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7513.0</td>\n",
       "      <td>Sales &amp; Marketing</td>\n",
       "      <td>region_19</td>\n",
       "      <td>Bachelor's</td>\n",
       "      <td>m</td>\n",
       "      <td>sourcing</td>\n",
       "      <td>1.0</td>\n",
       "      <td>34.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>50.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2542.0</td>\n",
       "      <td>Sales &amp; Marketing</td>\n",
       "      <td>region_23</td>\n",
       "      <td>Bachelor's</td>\n",
       "      <td>m</td>\n",
       "      <td>other</td>\n",
       "      <td>2.0</td>\n",
       "      <td>39.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>50.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>48945.0</td>\n",
       "      <td>Technology</td>\n",
       "      <td>region_26</td>\n",
       "      <td>Bachelor's</td>\n",
       "      <td>m</td>\n",
       "      <td>other</td>\n",
       "      <td>1.0</td>\n",
       "      <td>45.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>73.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9995</th>\n",
       "      <td>14934.0</td>\n",
       "      <td>Procurement</td>\n",
       "      <td>region_13</td>\n",
       "      <td>Master's &amp; above</td>\n",
       "      <td>f</td>\n",
       "      <td>other</td>\n",
       "      <td>1.0</td>\n",
       "      <td>37.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>71.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9996</th>\n",
       "      <td>22040.0</td>\n",
       "      <td>Sales &amp; Marketing</td>\n",
       "      <td>region_33</td>\n",
       "      <td>Master's &amp; above</td>\n",
       "      <td>m</td>\n",
       "      <td>sourcing</td>\n",
       "      <td>1.0</td>\n",
       "      <td>39.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>48.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9997</th>\n",
       "      <td>14188.0</td>\n",
       "      <td>Finance</td>\n",
       "      <td>region_13</td>\n",
       "      <td>Master's &amp; above</td>\n",
       "      <td>f</td>\n",
       "      <td>sourcing</td>\n",
       "      <td>1.0</td>\n",
       "      <td>33.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>58.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9998</th>\n",
       "      <td>73566.0</td>\n",
       "      <td>Operations</td>\n",
       "      <td>region_28</td>\n",
       "      <td>Master's &amp; above</td>\n",
       "      <td>m</td>\n",
       "      <td>other</td>\n",
       "      <td>1.0</td>\n",
       "      <td>32.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>57.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9999</th>\n",
       "      <td>21372.0</td>\n",
       "      <td>Procurement</td>\n",
       "      <td>region_13</td>\n",
       "      <td>Bachelor's</td>\n",
       "      <td>f</td>\n",
       "      <td>sourcing</td>\n",
       "      <td>1.0</td>\n",
       "      <td>32.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>71.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10000 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      employee_id         department     region         education gender  \\\n",
       "0         65438.0  Sales & Marketing   region_7  Master's & above      f   \n",
       "1         65141.0         Operations  region_22        Bachelor's      m   \n",
       "2          7513.0  Sales & Marketing  region_19        Bachelor's      m   \n",
       "3          2542.0  Sales & Marketing  region_23        Bachelor's      m   \n",
       "4         48945.0         Technology  region_26        Bachelor's      m   \n",
       "...           ...                ...        ...               ...    ...   \n",
       "9995      14934.0        Procurement  region_13  Master's & above      f   \n",
       "9996      22040.0  Sales & Marketing  region_33  Master's & above      m   \n",
       "9997      14188.0            Finance  region_13  Master's & above      f   \n",
       "9998      73566.0         Operations  region_28  Master's & above      m   \n",
       "9999      21372.0        Procurement  region_13        Bachelor's      f   \n",
       "\n",
       "     recruitment_channel  no_of_trainings   age  previous_year_rating  \\\n",
       "0               sourcing              1.0  35.0                   5.0   \n",
       "1                  other              1.0  30.0                   5.0   \n",
       "2               sourcing              1.0  34.0                   3.0   \n",
       "3                  other              2.0  39.0                   1.0   \n",
       "4                  other              1.0  45.0                   3.0   \n",
       "...                  ...              ...   ...                   ...   \n",
       "9995               other              1.0  37.0                   4.0   \n",
       "9996            sourcing              1.0  39.0                   3.0   \n",
       "9997            sourcing              1.0  33.0                   4.0   \n",
       "9998               other              1.0  32.0                   4.0   \n",
       "9999            sourcing              1.0  32.0                   3.0   \n",
       "\n",
       "      length_of_service  KPIs_met >80%  awards_won?  avg_training_score  \\\n",
       "0                   8.0            1.0          0.0                49.0   \n",
       "1                   4.0            0.0          0.0                60.0   \n",
       "2                   7.0            0.0          0.0                50.0   \n",
       "3                  10.0            0.0          0.0                50.0   \n",
       "4                   2.0            0.0          0.0                73.0   \n",
       "...                 ...            ...          ...                 ...   \n",
       "9995                7.0            1.0          0.0                71.0   \n",
       "9996                7.0            0.0          0.0                48.0   \n",
       "9997                4.0            1.0          0.0                58.0   \n",
       "9998                4.0            1.0          0.0                57.0   \n",
       "9999                6.0            0.0          0.0                71.0   \n",
       "\n",
       "      is_promoted  \n",
       "0             0.0  \n",
       "1             0.0  \n",
       "2             0.0  \n",
       "3             0.0  \n",
       "4             0.0  \n",
       "...           ...  \n",
       "9995          0.0  \n",
       "9996          0.0  \n",
       "9997          0.0  \n",
       "9998          1.0  \n",
       "9999          0.0  \n",
       "\n",
       "[10000 rows x 14 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "imputer = IterativeDataImputer(\n",
    "    df=dataset,\n",
    "    col_impute=None,\n",
    "    enable_encoder=True,\n",
    "    iterative_params={\n",
    "        'estimator': RandomForestRegressor(),\n",
    "        'missing_values': np.nan,\n",
    "        'sample_posterior': False,\n",
    "        'max_iter': 10,\n",
    "        'tol': 1e-3,\n",
    "        'n_nearest_features': None,\n",
    "        'initial_strategy': 'mean',\n",
    "        'imputation_order': 'ascending',\n",
    "        'skip_complete': False,\n",
    "        'min_value': -np.inf,\n",
    "        'max_value': np.inf,\n",
    "        'random_state': None}\n",
    ")\n",
    "imputer.fit()\n",
    "new_df = imputer.transform(dataset)\n",
    "new_df\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note that using the encoder before the imputation of **education** column isn't always guaranteed to reverse transformed as it now includes new imputed values that the encoder can't always map back to categorical data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "employee_id             False\n",
      "department              False\n",
      "region                  False\n",
      "education               False\n",
      "gender                  False\n",
      "recruitment_channel     False\n",
      "no_of_trainings         False\n",
      "age                     False\n",
      "previous_year_rating    False\n",
      "length_of_service       False\n",
      "KPIs_met >80%           False\n",
      "awards_won?             False\n",
      "avg_training_score      False\n",
      "is_promoted             False\n",
      "dtype: bool\n"
     ]
    }
   ],
   "source": [
    "print(new_df.isna().any())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now that the missing values have been filled, we can fit a model to the data. Here we do so using a decision tree..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No columns specified for encoding. These columns have been automatically identfied as the following:\n",
      "['department', 'region', 'education', 'gender', 'recruitment_channel']\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.879"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from raimitigations.dataprocessing import EncoderOrdinal\n",
    "\n",
    "encode = EncoderOrdinal(\n",
    "    df=new_df,\n",
    "    col_encode=None\n",
    ")\n",
    "encode.fit()\n",
    "new_df = encode.transform(new_df)\n",
    "\n",
    "estimator = DecisionTreeClassifier(max_features=\"sqrt\", random_state=0)\n",
    "X = new_df.drop(columns=['is_promoted', 'employee_id'])\n",
    "Y = new_df['is_promoted']\n",
    "\n",
    "train_X, test_X, train_y, test_y = train_test_split(X, Y, test_size=0.2, random_state=0, stratify=Y)\n",
    "\n",
    "estimator.fit(train_X, train_y)\n",
    "estimator.score(test_X, test_y)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Handling a DataFrame without column names\n",
    "\n",
    "Even if the dataset contains no header columns, we can perform the same operations, instead with the column index. The next few cells will demonstrate how to do this."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>11</th>\n",
       "      <th>12</th>\n",
       "      <th>13</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>65438</td>\n",
       "      <td>Sales &amp; Marketing</td>\n",
       "      <td>region_7</td>\n",
       "      <td>Master's &amp; above</td>\n",
       "      <td>f</td>\n",
       "      <td>sourcing</td>\n",
       "      <td>1</td>\n",
       "      <td>35</td>\n",
       "      <td>5.0</td>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>49</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>65141</td>\n",
       "      <td>Operations</td>\n",
       "      <td>region_22</td>\n",
       "      <td>Bachelor's</td>\n",
       "      <td>m</td>\n",
       "      <td>other</td>\n",
       "      <td>1</td>\n",
       "      <td>30</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>60</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7513</td>\n",
       "      <td>Sales &amp; Marketing</td>\n",
       "      <td>region_19</td>\n",
       "      <td>Bachelor's</td>\n",
       "      <td>m</td>\n",
       "      <td>sourcing</td>\n",
       "      <td>1</td>\n",
       "      <td>34</td>\n",
       "      <td>3.0</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>50</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2542</td>\n",
       "      <td>Sales &amp; Marketing</td>\n",
       "      <td>region_23</td>\n",
       "      <td>Bachelor's</td>\n",
       "      <td>m</td>\n",
       "      <td>other</td>\n",
       "      <td>2</td>\n",
       "      <td>39</td>\n",
       "      <td>1.0</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>50</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>48945</td>\n",
       "      <td>Technology</td>\n",
       "      <td>region_26</td>\n",
       "      <td>Bachelor's</td>\n",
       "      <td>m</td>\n",
       "      <td>other</td>\n",
       "      <td>1</td>\n",
       "      <td>45</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>73</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9995</th>\n",
       "      <td>14934</td>\n",
       "      <td>Procurement</td>\n",
       "      <td>region_13</td>\n",
       "      <td>Master's &amp; above</td>\n",
       "      <td>f</td>\n",
       "      <td>other</td>\n",
       "      <td>1</td>\n",
       "      <td>37</td>\n",
       "      <td>4.0</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>71</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9996</th>\n",
       "      <td>22040</td>\n",
       "      <td>Sales &amp; Marketing</td>\n",
       "      <td>region_33</td>\n",
       "      <td>Master's &amp; above</td>\n",
       "      <td>m</td>\n",
       "      <td>sourcing</td>\n",
       "      <td>1</td>\n",
       "      <td>39</td>\n",
       "      <td>3.0</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>48</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9997</th>\n",
       "      <td>14188</td>\n",
       "      <td>Finance</td>\n",
       "      <td>region_13</td>\n",
       "      <td>Master's &amp; above</td>\n",
       "      <td>f</td>\n",
       "      <td>sourcing</td>\n",
       "      <td>1</td>\n",
       "      <td>33</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>58</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9998</th>\n",
       "      <td>73566</td>\n",
       "      <td>Operations</td>\n",
       "      <td>region_28</td>\n",
       "      <td>Master's &amp; above</td>\n",
       "      <td>m</td>\n",
       "      <td>other</td>\n",
       "      <td>1</td>\n",
       "      <td>32</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>57</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9999</th>\n",
       "      <td>21372</td>\n",
       "      <td>Procurement</td>\n",
       "      <td>region_13</td>\n",
       "      <td>Bachelor's</td>\n",
       "      <td>f</td>\n",
       "      <td>sourcing</td>\n",
       "      <td>1</td>\n",
       "      <td>32</td>\n",
       "      <td>3.0</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>71</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10000 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         0                  1          2                 3  4         5   6   \\\n",
       "0     65438  Sales & Marketing   region_7  Master's & above  f  sourcing   1   \n",
       "1     65141         Operations  region_22        Bachelor's  m     other   1   \n",
       "2      7513  Sales & Marketing  region_19        Bachelor's  m  sourcing   1   \n",
       "3      2542  Sales & Marketing  region_23        Bachelor's  m     other   2   \n",
       "4     48945         Technology  region_26        Bachelor's  m     other   1   \n",
       "...     ...                ...        ...               ... ..       ...  ..   \n",
       "9995  14934        Procurement  region_13  Master's & above  f     other   1   \n",
       "9996  22040  Sales & Marketing  region_33  Master's & above  m  sourcing   1   \n",
       "9997  14188            Finance  region_13  Master's & above  f  sourcing   1   \n",
       "9998  73566         Operations  region_28  Master's & above  m     other   1   \n",
       "9999  21372        Procurement  region_13        Bachelor's  f  sourcing   1   \n",
       "\n",
       "      7    8   9   10  11  12  13  \n",
       "0     35  5.0   8   1   0  49   0  \n",
       "1     30  5.0   4   0   0  60   0  \n",
       "2     34  3.0   7   0   0  50   0  \n",
       "3     39  1.0  10   0   0  50   0  \n",
       "4     45  3.0   2   0   0  73   0  \n",
       "...   ..  ...  ..  ..  ..  ..  ..  \n",
       "9995  37  4.0   7   1   0  71   0  \n",
       "9996  39  3.0   7   0   0  48   0  \n",
       "9997  33  4.0   4   1   0  58   0  \n",
       "9998  32  4.0   4   1   0  57   1  \n",
       "9999  32  3.0   6   0   0  71   0  \n",
       "\n",
       "[10000 rows x 14 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset = pd.read_csv(data_dir + 'hr_promotion/train.csv', header=None, skiprows=1)\n",
    "dataset = dataset[:10000]\n",
    "dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0     False\n",
      "1     False\n",
      "2     False\n",
      "3      True\n",
      "4     False\n",
      "5     False\n",
      "6     False\n",
      "7     False\n",
      "8      True\n",
      "9     False\n",
      "10    False\n",
      "11    False\n",
      "12    False\n",
      "13    False\n",
      "dtype: bool\n",
      "[\"Master's & above\" \"Bachelor's\" nan 'Below Secondary']\n",
      "[ 5.  3.  1.  4. nan  2.]\n"
     ]
    }
   ],
   "source": [
    "print(dataset.isna().any())\n",
    "print(dataset.iloc[:, 3].unique())\n",
    "print(dataset.iloc[:, 8].unique())"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Without Enabling Encoding\n",
    "\n",
    "Using the `sklearn_obj` parameter, we have the option to pass a pre-defined `sklearn.impute.IterativeImputer` object. If the latter is used, `iterative_params` will be overwritten."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "WARNING: Categorical columns will be excluded from the iterative imputation process.\n",
      "If you'd like to include these columns, you need to set 'enable_encoder'=True.\n",
      "If you'd like to use a different type of encoding before imputation, consider using the Pipeline class and call your own encoder before calling this subclass for imputation.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>11</th>\n",
       "      <th>12</th>\n",
       "      <th>13</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>65438.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>35.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>49.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Sales &amp; Marketing</td>\n",
       "      <td>region_7</td>\n",
       "      <td>f</td>\n",
       "      <td>sourcing</td>\n",
       "      <td>Master's &amp; above</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>65141.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>30.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>60.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Operations</td>\n",
       "      <td>region_22</td>\n",
       "      <td>m</td>\n",
       "      <td>other</td>\n",
       "      <td>Bachelor's</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7513.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>34.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>50.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Sales &amp; Marketing</td>\n",
       "      <td>region_19</td>\n",
       "      <td>m</td>\n",
       "      <td>sourcing</td>\n",
       "      <td>Bachelor's</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2542.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>39.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>50.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Sales &amp; Marketing</td>\n",
       "      <td>region_23</td>\n",
       "      <td>m</td>\n",
       "      <td>other</td>\n",
       "      <td>Bachelor's</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>48945.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>45.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>73.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Technology</td>\n",
       "      <td>region_26</td>\n",
       "      <td>m</td>\n",
       "      <td>other</td>\n",
       "      <td>Bachelor's</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9995</th>\n",
       "      <td>14934.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>37.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>71.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Procurement</td>\n",
       "      <td>region_13</td>\n",
       "      <td>f</td>\n",
       "      <td>other</td>\n",
       "      <td>Master's &amp; above</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9996</th>\n",
       "      <td>22040.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>39.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>48.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Sales &amp; Marketing</td>\n",
       "      <td>region_33</td>\n",
       "      <td>m</td>\n",
       "      <td>sourcing</td>\n",
       "      <td>Master's &amp; above</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9997</th>\n",
       "      <td>14188.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>33.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>58.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Finance</td>\n",
       "      <td>region_13</td>\n",
       "      <td>f</td>\n",
       "      <td>sourcing</td>\n",
       "      <td>Master's &amp; above</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9998</th>\n",
       "      <td>73566.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>32.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>57.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Operations</td>\n",
       "      <td>region_28</td>\n",
       "      <td>m</td>\n",
       "      <td>other</td>\n",
       "      <td>Master's &amp; above</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9999</th>\n",
       "      <td>21372.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>32.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>71.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Procurement</td>\n",
       "      <td>region_13</td>\n",
       "      <td>f</td>\n",
       "      <td>sourcing</td>\n",
       "      <td>Bachelor's</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10000 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            0    6     7    8     9   10   11    12   13                  1  \\\n",
       "0     65438.0  1.0  35.0  5.0   8.0  1.0  0.0  49.0  0.0  Sales & Marketing   \n",
       "1     65141.0  1.0  30.0  5.0   4.0  0.0  0.0  60.0  0.0         Operations   \n",
       "2      7513.0  1.0  34.0  3.0   7.0  0.0  0.0  50.0  0.0  Sales & Marketing   \n",
       "3      2542.0  2.0  39.0  1.0  10.0  0.0  0.0  50.0  0.0  Sales & Marketing   \n",
       "4     48945.0  1.0  45.0  3.0   2.0  0.0  0.0  73.0  0.0         Technology   \n",
       "...       ...  ...   ...  ...   ...  ...  ...   ...  ...                ...   \n",
       "9995  14934.0  1.0  37.0  4.0   7.0  1.0  0.0  71.0  0.0        Procurement   \n",
       "9996  22040.0  1.0  39.0  3.0   7.0  0.0  0.0  48.0  0.0  Sales & Marketing   \n",
       "9997  14188.0  1.0  33.0  4.0   4.0  1.0  0.0  58.0  0.0            Finance   \n",
       "9998  73566.0  1.0  32.0  4.0   4.0  1.0  0.0  57.0  1.0         Operations   \n",
       "9999  21372.0  1.0  32.0  3.0   6.0  0.0  0.0  71.0  0.0        Procurement   \n",
       "\n",
       "              2  4         5                 3  \n",
       "0      region_7  f  sourcing  Master's & above  \n",
       "1     region_22  m     other        Bachelor's  \n",
       "2     region_19  m  sourcing        Bachelor's  \n",
       "3     region_23  m     other        Bachelor's  \n",
       "4     region_26  m     other        Bachelor's  \n",
       "...         ... ..       ...               ...  \n",
       "9995  region_13  f     other  Master's & above  \n",
       "9996  region_33  m  sourcing  Master's & above  \n",
       "9997  region_13  f  sourcing  Master's & above  \n",
       "9998  region_28  m     other  Master's & above  \n",
       "9999  region_13  f  sourcing        Bachelor's  \n",
       "\n",
       "[10000 rows x 14 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.experimental import enable_iterative_imputer  # noqa # pylint: disable=unused-import\n",
    "from sklearn.impute import IterativeImputer\n",
    "\n",
    "imputer = IterativeDataImputer(\n",
    "    df=dataset,\n",
    "    col_impute=[8],\n",
    "    enable_encoder=False,\n",
    "    sklearn_obj=IterativeImputer(estimator=RandomForestRegressor(), random_state=100),\n",
    ")\n",
    "imputer.fit()\n",
    "new_df = imputer.transform(dataset)\n",
    "new_df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0     False\n",
      "6     False\n",
      "7     False\n",
      "8     False\n",
      "9     False\n",
      "10    False\n",
      "11    False\n",
      "12    False\n",
      "13    False\n",
      "1     False\n",
      "2     False\n",
      "4     False\n",
      "5     False\n",
      "3      True\n",
      "dtype: bool\n"
     ]
    }
   ],
   "source": [
    "print(new_df.isna().any())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### With Enabling Encoding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "WARNING: 'enable_encoder'=True and categorical columns will be encoded using ordinal encoding before applying the iterative imputation process.\n",
      "If you'd like to use a different type of encoding before imputation, consider using the Pipeline class and call your own encoder before calling this subclass for imputation.\n",
      "[IterativeImputer] Completing matrix with shape (10000, 14)\n",
      "[IterativeImputer] Change: 1.744667046565469, scaled tolerance: 78.297 \n",
      "[IterativeImputer] Early stopping criterion reached.\n",
      "No columns specified for imputation. These columns have been automatically identified at transform time:\n",
      "['3', '8']\n",
      "\n",
      "WARNING: Note that encoded columns are not guaranteed to reverse transform if they have imputed values.\n",
      "If you'd like to use a different type of encoding before imputation, consider using the Pipeline class and call your own encoder before calling this subclass.\n",
      "[IterativeImputer] Completing matrix with shape (10000, 14)\n",
      "\n",
      "Imputed categorical columns' reverse encoding transformation complete.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>11</th>\n",
       "      <th>12</th>\n",
       "      <th>13</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>65438.0</td>\n",
       "      <td>Sales &amp; Marketing</td>\n",
       "      <td>region_7</td>\n",
       "      <td>Master's &amp; above</td>\n",
       "      <td>f</td>\n",
       "      <td>sourcing</td>\n",
       "      <td>1.0</td>\n",
       "      <td>35.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>49.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>65141.0</td>\n",
       "      <td>Operations</td>\n",
       "      <td>region_22</td>\n",
       "      <td>Bachelor's</td>\n",
       "      <td>m</td>\n",
       "      <td>other</td>\n",
       "      <td>1.0</td>\n",
       "      <td>30.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>60.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7513.0</td>\n",
       "      <td>Sales &amp; Marketing</td>\n",
       "      <td>region_19</td>\n",
       "      <td>Bachelor's</td>\n",
       "      <td>m</td>\n",
       "      <td>sourcing</td>\n",
       "      <td>1.0</td>\n",
       "      <td>34.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>50.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2542.0</td>\n",
       "      <td>Sales &amp; Marketing</td>\n",
       "      <td>region_23</td>\n",
       "      <td>Bachelor's</td>\n",
       "      <td>m</td>\n",
       "      <td>other</td>\n",
       "      <td>2.0</td>\n",
       "      <td>39.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>50.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>48945.0</td>\n",
       "      <td>Technology</td>\n",
       "      <td>region_26</td>\n",
       "      <td>Bachelor's</td>\n",
       "      <td>m</td>\n",
       "      <td>other</td>\n",
       "      <td>1.0</td>\n",
       "      <td>45.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>73.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9995</th>\n",
       "      <td>14934.0</td>\n",
       "      <td>Procurement</td>\n",
       "      <td>region_13</td>\n",
       "      <td>Master's &amp; above</td>\n",
       "      <td>f</td>\n",
       "      <td>other</td>\n",
       "      <td>1.0</td>\n",
       "      <td>37.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>71.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9996</th>\n",
       "      <td>22040.0</td>\n",
       "      <td>Sales &amp; Marketing</td>\n",
       "      <td>region_33</td>\n",
       "      <td>Master's &amp; above</td>\n",
       "      <td>m</td>\n",
       "      <td>sourcing</td>\n",
       "      <td>1.0</td>\n",
       "      <td>39.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>48.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9997</th>\n",
       "      <td>14188.0</td>\n",
       "      <td>Finance</td>\n",
       "      <td>region_13</td>\n",
       "      <td>Master's &amp; above</td>\n",
       "      <td>f</td>\n",
       "      <td>sourcing</td>\n",
       "      <td>1.0</td>\n",
       "      <td>33.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>58.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9998</th>\n",
       "      <td>73566.0</td>\n",
       "      <td>Operations</td>\n",
       "      <td>region_28</td>\n",
       "      <td>Master's &amp; above</td>\n",
       "      <td>m</td>\n",
       "      <td>other</td>\n",
       "      <td>1.0</td>\n",
       "      <td>32.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>57.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9999</th>\n",
       "      <td>21372.0</td>\n",
       "      <td>Procurement</td>\n",
       "      <td>region_13</td>\n",
       "      <td>Bachelor's</td>\n",
       "      <td>f</td>\n",
       "      <td>sourcing</td>\n",
       "      <td>1.0</td>\n",
       "      <td>32.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>71.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10000 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            0                  1          2                 3  4         5  \\\n",
       "0     65438.0  Sales & Marketing   region_7  Master's & above  f  sourcing   \n",
       "1     65141.0         Operations  region_22        Bachelor's  m     other   \n",
       "2      7513.0  Sales & Marketing  region_19        Bachelor's  m  sourcing   \n",
       "3      2542.0  Sales & Marketing  region_23        Bachelor's  m     other   \n",
       "4     48945.0         Technology  region_26        Bachelor's  m     other   \n",
       "...       ...                ...        ...               ... ..       ...   \n",
       "9995  14934.0        Procurement  region_13  Master's & above  f     other   \n",
       "9996  22040.0  Sales & Marketing  region_33  Master's & above  m  sourcing   \n",
       "9997  14188.0            Finance  region_13  Master's & above  f  sourcing   \n",
       "9998  73566.0         Operations  region_28  Master's & above  m     other   \n",
       "9999  21372.0        Procurement  region_13        Bachelor's  f  sourcing   \n",
       "\n",
       "        6     7    8     9   10   11    12   13  \n",
       "0     1.0  35.0  5.0   8.0  1.0  0.0  49.0  0.0  \n",
       "1     1.0  30.0  5.0   4.0  0.0  0.0  60.0  0.0  \n",
       "2     1.0  34.0  3.0   7.0  0.0  0.0  50.0  0.0  \n",
       "3     2.0  39.0  1.0  10.0  0.0  0.0  50.0  0.0  \n",
       "4     1.0  45.0  3.0   2.0  0.0  0.0  73.0  0.0  \n",
       "...   ...   ...  ...   ...  ...  ...   ...  ...  \n",
       "9995  1.0  37.0  4.0   7.0  1.0  0.0  71.0  0.0  \n",
       "9996  1.0  39.0  3.0   7.0  0.0  0.0  48.0  0.0  \n",
       "9997  1.0  33.0  4.0   4.0  1.0  0.0  58.0  0.0  \n",
       "9998  1.0  32.0  4.0   4.0  1.0  0.0  57.0  1.0  \n",
       "9999  1.0  32.0  3.0   6.0  0.0  0.0  71.0  0.0  \n",
       "\n",
       "[10000 rows x 14 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "imputer = IterativeDataImputer(\n",
    "    df=dataset,\n",
    "    col_impute=None,\n",
    "    enable_encoder=True,\n",
    "    iterative_params={\n",
    "        'estimator': RandomForestRegressor(),\n",
    "        'missing_values': np.nan,\n",
    "        'sample_posterior': False,\n",
    "        'max_iter': 10,\n",
    "        'tol': 1e-3,\n",
    "        'n_nearest_features': None,\n",
    "        'initial_strategy': 'mean',\n",
    "        'imputation_order': 'ascending',\n",
    "        'skip_complete': False,\n",
    "        'min_value': -np.inf,\n",
    "        'max_value': np.inf,\n",
    "        'random_state': None}\n",
    ")\n",
    "imputer.fit()\n",
    "new_df = imputer.transform(dataset)\n",
    "new_df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0     False\n",
      "1     False\n",
      "2     False\n",
      "3     False\n",
      "4     False\n",
      "5     False\n",
      "6     False\n",
      "7     False\n",
      "8     False\n",
      "9     False\n",
      "10    False\n",
      "11    False\n",
      "12    False\n",
      "13    False\n",
      "dtype: bool\n"
     ]
    }
   ],
   "source": [
    "print(new_df.isna().any())"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.9.13 ('rai')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "239972f3e2cce5deeefd34fe5d6dd927057b08952d864776048f8db51d1a831a"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}