Data Acquisition.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-25T10:13:46.671395Z",
     "start_time": "2022-05-25T10:13:46.656868Z"
    }
   },
   "outputs": [],
   "source": [
    "import time\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import confusion_matrix, classification_report, accuracy_score\n",
    "from sklearn.decomposition import TruncatedSVD\n",
    "from imblearn.over_sampling import SMOTE, RandomOverSampler\n",
    "from imblearn.under_sampling import RandomUnderSampler\n",
    "from imblearn.combine import SMOTEENN\n",
    "from imblearn.pipeline import Pipeline, make_pipeline\n",
    "from tensorflow.keras.layers import Dense, Dropout\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.callbacks import EarlyStopping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-25T10:12:15.408807Z",
     "start_time": "2022-05-25T10:12:10.174186Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>posts</th>\n",
       "      <th>type</th>\n",
       "      <th>I/E</th>\n",
       "      <th>dominant_auxiliary</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>know intj tool use interaction people excuse a...</td>\n",
       "      <td>INTJ</td>\n",
       "      <td>I</td>\n",
       "      <td>NT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>rap music ehh opp yeah know valid well know fa...</td>\n",
       "      <td>INTJ</td>\n",
       "      <td>I</td>\n",
       "      <td>NT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>preferably p hd low except wew lad video p min...</td>\n",
       "      <td>INTJ</td>\n",
       "      <td>I</td>\n",
       "      <td>NT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>drink like wish could drink red wine give head...</td>\n",
       "      <td>INTJ</td>\n",
       "      <td>I</td>\n",
       "      <td>NT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>space program ah bad deal meing freelance max ...</td>\n",
       "      <td>INTJ</td>\n",
       "      <td>I</td>\n",
       "      <td>NT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>106062</th>\n",
       "      <td>stay frustrate world life want take long nap w...</td>\n",
       "      <td>INFP</td>\n",
       "      <td>I</td>\n",
       "      <td>FN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>106063</th>\n",
       "      <td>fizzle around time mention sure mistake thing ...</td>\n",
       "      <td>INFP</td>\n",
       "      <td>I</td>\n",
       "      <td>FN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>106064</th>\n",
       "      <td>schedule modify hey w intp strong wing underst...</td>\n",
       "      <td>INFP</td>\n",
       "      <td>I</td>\n",
       "      <td>FN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>106065</th>\n",
       "      <td>enfj since january busy schedule able spend li...</td>\n",
       "      <td>INFP</td>\n",
       "      <td>I</td>\n",
       "      <td>FN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>106066</th>\n",
       "      <td>feel like men good problem tell parent want te...</td>\n",
       "      <td>INFP</td>\n",
       "      <td>I</td>\n",
       "      <td>FN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>106067 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    posts  type I/E  \\\n",
       "0       know intj tool use interaction people excuse a...  INTJ   I   \n",
       "1       rap music ehh opp yeah know valid well know fa...  INTJ   I   \n",
       "2       preferably p hd low except wew lad video p min...  INTJ   I   \n",
       "3       drink like wish could drink red wine give head...  INTJ   I   \n",
       "4       space program ah bad deal meing freelance max ...  INTJ   I   \n",
       "...                                                   ...   ...  ..   \n",
       "106062  stay frustrate world life want take long nap w...  INFP   I   \n",
       "106063  fizzle around time mention sure mistake thing ...  INFP   I   \n",
       "106064  schedule modify hey w intp strong wing underst...  INFP   I   \n",
       "106065  enfj since january busy schedule able spend li...  INFP   I   \n",
       "106066  feel like men good problem tell parent want te...  INFP   I   \n",
       "\n",
       "       dominant_auxiliary  \n",
       "0                      NT  \n",
       "1                      NT  \n",
       "2                      NT  \n",
       "3                      NT  \n",
       "4                      NT  \n",
       "...                   ...  \n",
       "106062                 FN  \n",
       "106063                 FN  \n",
       "106064                 FN  \n",
       "106065                 FN  \n",
       "106066                 FN  \n",
       "\n",
       "[106067 rows x 4 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_csv(\"data/MBTI_500.csv\")\n",
    "data['I/E'] = data['type'].str[0]\n",
    "data.loc[((data['I/E'] == 'I') & (data['type'].str[3] == 'J')) | ((data['I/E'] == 'E') & (data['type'].str[3] == 'P')), 'dominant_auxiliary'] = data.loc[((data['I/E'] == 'I') & (data['type'].str[3] == 'J')) | ((data['I/E'] == 'E') & (data['type'].str[3] == 'P')), \"type\"].str[1:3]\n",
    "data.loc[data['dominant_auxiliary'].isnull(), 'dominant_auxiliary'] = data.loc[data['dominant_auxiliary'].isnull(), \"type\"].str[-2:-4:-1]\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-25T10:13:31.319328Z",
     "start_time": "2022-05-25T10:12:18.494183Z"
    }
   },
   "outputs": [],
   "source": [
    "vectorizer = TfidfVectorizer()\n",
    "x_tfidf = vectorizer.fit_transform(data['posts'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-25T10:14:56.980702Z",
     "start_time": "2022-05-25T10:13:49.733706Z"
    }
   },
   "outputs": [],
   "source": [
    "#Perform undersampling\n",
    "under = RandomUnderSampler(sampling_strategy = dict(np.minimum(data['type'].value_counts(), 2000)))\n",
    "x, y = under.fit_resample(x_tfidf, data['type'])\n",
    "\n",
    "#Dimesionality reduction\n",
    "svd = TruncatedSVD(n_components=250)\n",
    "x = svd.fit_transform(x)\n",
    "\n",
    "x_train, x_val, y_train, y_val = train_test_split(x, y, test_size=0.4)\n",
    "x_val, x_test, y_val, y_test = train_test_split(x_val, y_val, test_size=0.75)\n",
    "\n",
    "#Oversampling\n",
    "smote = SMOTE()\n",
    "x_train, y_train = smote.fit_resample(x_train, y_train)\n",
    "x_val, y_val = smote.fit_resample(x_val, y_val)\n",
    "x_test, y_test = smote.fit_resample(x_test, y_test)\n",
    "y_train = pd.get_dummies(y_train)\n",
    "y_val = pd.get_dummies(y_val)\n",
    "n_cols = x_train.shape[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-25T10:19:33.902417Z",
     "start_time": "2022-05-25T10:15:17.271338Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100\n",
      "612/612 [==============================] - 4s 5ms/step - loss: 2.7692 - accuracy: 0.0991 - val_loss: 2.7618 - val_accuracy: 0.1910\n",
      "Epoch 2/100\n",
      "612/612 [==============================] - 3s 5ms/step - loss: 2.7570 - accuracy: 0.1926 - val_loss: 2.7489 - val_accuracy: 0.3267\n",
      "Epoch 3/100\n",
      "612/612 [==============================] - 3s 5ms/step - loss: 2.7439 - accuracy: 0.2679 - val_loss: 2.7335 - val_accuracy: 0.4035\n",
      "Epoch 4/100\n",
      "612/612 [==============================] - 3s 5ms/step - loss: 2.7273 - accuracy: 0.2994 - val_loss: 2.7133 - val_accuracy: 0.4826\n",
      "Epoch 5/100\n",
      "612/612 [==============================] - 3s 5ms/step - loss: 2.7045 - accuracy: 0.3324 - val_loss: 2.6857 - val_accuracy: 0.4689\n",
      "Epoch 6/100\n",
      "612/612 [==============================] - 4s 6ms/step - loss: 2.6743 - accuracy: 0.3458 - val_loss: 2.6485 - val_accuracy: 0.5328\n",
      "Epoch 7/100\n",
      "612/612 [==============================] - 4s 6ms/step - loss: 2.6341 - accuracy: 0.3660 - val_loss: 2.6013 - val_accuracy: 0.4988\n",
      "Epoch 8/100\n",
      "612/612 [==============================] - 4s 6ms/step - loss: 2.5887 - accuracy: 0.3859 - val_loss: 2.5483 - val_accuracy: 0.5267\n",
      "Epoch 9/100\n",
      "612/612 [==============================] - 5s 8ms/step - loss: 2.5375 - accuracy: 0.3933 - val_loss: 2.4930 - val_accuracy: 0.5797\n",
      "Epoch 10/100\n",
      "612/612 [==============================] - 5s 8ms/step - loss: 2.4840 - accuracy: 0.4041 - val_loss: 2.4345 - val_accuracy: 0.5962\n",
      "Epoch 11/100\n",
      "612/612 [==============================] - 5s 8ms/step - loss: 2.4164 - accuracy: 0.4221 - val_loss: 2.3672 - val_accuracy: 0.5765\n",
      "Epoch 12/100\n",
      "612/612 [==============================] - 4s 7ms/step - loss: 2.3426 - accuracy: 0.4392 - val_loss: 2.2894 - val_accuracy: 0.5843\n",
      "Epoch 13/100\n",
      "612/612 [==============================] - 4s 6ms/step - loss: 2.2586 - accuracy: 0.4595 - val_loss: 2.2032 - val_accuracy: 0.6177\n",
      "Epoch 14/100\n",
      "612/612 [==============================] - 5s 8ms/step - loss: 2.1763 - accuracy: 0.4833 - val_loss: 2.1140 - val_accuracy: 0.6459\n",
      "Epoch 15/100\n",
      "612/612 [==============================] - 4s 7ms/step - loss: 2.0856 - accuracy: 0.5044 - val_loss: 2.0203 - val_accuracy: 0.6477\n",
      "Epoch 16/100\n",
      "612/612 [==============================] - 4s 7ms/step - loss: 1.9951 - accuracy: 0.5246 - val_loss: 1.9289 - val_accuracy: 0.6750\n",
      "Epoch 17/100\n",
      "612/612 [==============================] - 4s 7ms/step - loss: 1.9073 - accuracy: 0.5507 - val_loss: 1.8380 - val_accuracy: 0.6683\n",
      "Epoch 18/100\n",
      "612/612 [==============================] - 4s 6ms/step - loss: 1.8258 - accuracy: 0.5656 - val_loss: 1.7465 - val_accuracy: 0.6808\n",
      "Epoch 19/100\n",
      "612/612 [==============================] - 4s 6ms/step - loss: 1.7460 - accuracy: 0.5817 - val_loss: 1.6624 - val_accuracy: 0.6846\n",
      "Epoch 20/100\n",
      "612/612 [==============================] - 4s 6ms/step - loss: 1.6653 - accuracy: 0.5973 - val_loss: 1.5728 - val_accuracy: 0.7064\n",
      "Epoch 21/100\n",
      "612/612 [==============================] - 4s 6ms/step - loss: 1.5893 - accuracy: 0.6118 - val_loss: 1.4923 - val_accuracy: 0.7125\n",
      "Epoch 22/100\n",
      "612/612 [==============================] - 4s 7ms/step - loss: 1.5187 - accuracy: 0.6234 - val_loss: 1.4185 - val_accuracy: 0.7247\n",
      "Epoch 23/100\n",
      "612/612 [==============================] - 4s 6ms/step - loss: 1.4550 - accuracy: 0.6413 - val_loss: 1.3481 - val_accuracy: 0.7378\n",
      "Epoch 24/100\n",
      "612/612 [==============================] - 4s 6ms/step - loss: 1.3941 - accuracy: 0.6491 - val_loss: 1.2876 - val_accuracy: 0.7285\n",
      "Epoch 25/100\n",
      "612/612 [==============================] - 4s 6ms/step - loss: 1.3456 - accuracy: 0.6555 - val_loss: 1.2222 - val_accuracy: 0.7535\n",
      "Epoch 26/100\n",
      "612/612 [==============================] - 4s 7ms/step - loss: 1.2971 - accuracy: 0.6698 - val_loss: 1.1673 - val_accuracy: 0.7578\n",
      "Epoch 27/100\n",
      "612/612 [==============================] - 4s 7ms/step - loss: 1.2494 - accuracy: 0.6757 - val_loss: 1.1232 - val_accuracy: 0.7538\n",
      "Epoch 28/100\n",
      "612/612 [==============================] - 4s 6ms/step - loss: 1.2143 - accuracy: 0.6830 - val_loss: 1.0798 - val_accuracy: 0.7599\n",
      "Epoch 29/100\n",
      "612/612 [==============================] - 4s 6ms/step - loss: 1.1764 - accuracy: 0.6878 - val_loss: 1.0415 - val_accuracy: 0.7744\n",
      "Epoch 30/100\n",
      "612/612 [==============================] - 4s 6ms/step - loss: 1.1447 - accuracy: 0.6978 - val_loss: 1.0091 - val_accuracy: 0.7779\n",
      "Epoch 31/100\n",
      "612/612 [==============================] - 4s 6ms/step - loss: 1.1102 - accuracy: 0.7047 - val_loss: 0.9781 - val_accuracy: 0.7863\n",
      "Epoch 32/100\n",
      "612/612 [==============================] - 4s 6ms/step - loss: 1.0893 - accuracy: 0.7057 - val_loss: 0.9467 - val_accuracy: 0.7788\n",
      "Epoch 33/100\n",
      "612/612 [==============================] - 4s 6ms/step - loss: 1.0572 - accuracy: 0.7141 - val_loss: 0.9246 - val_accuracy: 0.7881\n",
      "Epoch 34/100\n",
      "612/612 [==============================] - 4s 6ms/step - loss: 1.0299 - accuracy: 0.7210 - val_loss: 0.9007 - val_accuracy: 0.7907\n",
      "Epoch 35/100\n",
      "612/612 [==============================] - 4s 6ms/step - loss: 1.0161 - accuracy: 0.7246 - val_loss: 0.8730 - val_accuracy: 0.7922\n",
      "Epoch 36/100\n",
      "612/612 [==============================] - 4s 6ms/step - loss: 0.9976 - accuracy: 0.7268 - val_loss: 0.8610 - val_accuracy: 0.7939\n",
      "Epoch 37/100\n",
      "612/612 [==============================] - 4s 7ms/step - loss: 0.9754 - accuracy: 0.7303 - val_loss: 0.8288 - val_accuracy: 0.8078\n",
      "Epoch 38/100\n",
      "612/612 [==============================] - 4s 7ms/step - loss: 0.9617 - accuracy: 0.7379 - val_loss: 0.8235 - val_accuracy: 0.8017\n",
      "Epoch 39/100\n",
      "612/612 [==============================] - 4s 6ms/step - loss: 0.9469 - accuracy: 0.7375 - val_loss: 0.7994 - val_accuracy: 0.8073\n",
      "Epoch 40/100\n",
      "612/612 [==============================] - 4s 6ms/step - loss: 0.9289 - accuracy: 0.7436 - val_loss: 0.7981 - val_accuracy: 0.8044\n",
      "Epoch 41/100\n",
      "612/612 [==============================] - 4s 6ms/step - loss: 0.9204 - accuracy: 0.7431 - val_loss: 0.7845 - val_accuracy: 0.8058\n",
      "Epoch 42/100\n",
      "612/612 [==============================] - 4s 6ms/step - loss: 0.9068 - accuracy: 0.7456 - val_loss: 0.7715 - val_accuracy: 0.8105\n",
      "Epoch 43/100\n",
      "612/612 [==============================] - 4s 6ms/step - loss: 0.8882 - accuracy: 0.7513 - val_loss: 0.7520 - val_accuracy: 0.8140\n",
      "Epoch 44/100\n",
      "612/612 [==============================] - 4s 6ms/step - loss: 0.8860 - accuracy: 0.7527 - val_loss: 0.7395 - val_accuracy: 0.8166\n",
      "Epoch 45/100\n",
      "612/612 [==============================] - 4s 6ms/step - loss: 0.8645 - accuracy: 0.7579 - val_loss: 0.7238 - val_accuracy: 0.8186\n",
      "Epoch 46/100\n",
      "612/612 [==============================] - 4s 6ms/step - loss: 0.8620 - accuracy: 0.7584 - val_loss: 0.7189 - val_accuracy: 0.8163\n",
      "Epoch 47/100\n",
      "612/612 [==============================] - 4s 6ms/step - loss: 0.8488 - accuracy: 0.7634 - val_loss: 0.7179 - val_accuracy: 0.8160\n",
      "Epoch 48/100\n",
      "612/612 [==============================] - 4s 7ms/step - loss: 0.8438 - accuracy: 0.7636 - val_loss: 0.7101 - val_accuracy: 0.8145\n",
      "Epoch 49/100\n",
      "612/612 [==============================] - 4s 6ms/step - loss: 0.8294 - accuracy: 0.7671 - val_loss: 0.7040 - val_accuracy: 0.8233\n",
      "Epoch 50/100\n",
      "612/612 [==============================] - 4s 6ms/step - loss: 0.8238 - accuracy: 0.7669 - val_loss: 0.6922 - val_accuracy: 0.8201\n",
      "Epoch 51/100\n",
      "612/612 [==============================] - 4s 7ms/step - loss: 0.8180 - accuracy: 0.7688 - val_loss: 0.6868 - val_accuracy: 0.8244\n",
      "Epoch 52/100\n",
      "612/612 [==============================] - 4s 7ms/step - loss: 0.8080 - accuracy: 0.7699 - val_loss: 0.6827 - val_accuracy: 0.8221\n",
      "Epoch 53/100\n",
      "612/612 [==============================] - 4s 7ms/step - loss: 0.8057 - accuracy: 0.7686 - val_loss: 0.6796 - val_accuracy: 0.8209\n",
      "Epoch 54/100\n",
      "612/612 [==============================] - 4s 6ms/step - loss: 0.8033 - accuracy: 0.7707 - val_loss: 0.6772 - val_accuracy: 0.8203\n",
      "Epoch 55/100\n",
      "612/612 [==============================] - 4s 6ms/step - loss: 0.7940 - accuracy: 0.7749 - val_loss: 0.6814 - val_accuracy: 0.8122\n",
      "Epoch 56/100\n",
      "612/612 [==============================] - 4s 6ms/step - loss: 0.7938 - accuracy: 0.7726 - val_loss: 0.6665 - val_accuracy: 0.8212\n",
      "Epoch 57/100\n",
      "612/612 [==============================] - 4s 6ms/step - loss: 0.7824 - accuracy: 0.7762 - val_loss: 0.6605 - val_accuracy: 0.8203\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 58/100\n",
      "612/612 [==============================] - 5s 8ms/step - loss: 0.7800 - accuracy: 0.7729 - val_loss: 0.6617 - val_accuracy: 0.8247\n",
      "Epoch 59/100\n",
      "612/612 [==============================] - 5s 8ms/step - loss: 0.7686 - accuracy: 0.7792 - val_loss: 0.6521 - val_accuracy: 0.8250\n",
      "Epoch 60/100\n",
      "612/612 [==============================] - 4s 7ms/step - loss: 0.7666 - accuracy: 0.7772 - val_loss: 0.6561 - val_accuracy: 0.8233\n",
      "Epoch 61/100\n",
      "612/612 [==============================] - 4s 7ms/step - loss: 0.7531 - accuracy: 0.7827 - val_loss: 0.6497 - val_accuracy: 0.8241\n",
      "Epoch 62/100\n",
      "612/612 [==============================] - 4s 7ms/step - loss: 0.7552 - accuracy: 0.7851 - val_loss: 0.6490 - val_accuracy: 0.8233\n",
      "Epoch 63/100\n",
      "612/612 [==============================] - 4s 7ms/step - loss: 0.7485 - accuracy: 0.7837 - val_loss: 0.6567 - val_accuracy: 0.8201\n",
      "Epoch 64/100\n",
      "612/612 [==============================] - 4s 7ms/step - loss: 0.7551 - accuracy: 0.7808 - val_loss: 0.6523 - val_accuracy: 0.8244\n"
     ]
    }
   ],
   "source": [
    "#Initialise - 1 layer with 500 nodes\n",
    "model = Sequential()\n",
    "model.add(Dropout(rate=0.2))\n",
    "model.add(Dense(500, activation='relu', input_shape=(n_cols, )))\n",
    "model.add(Dropout(rate=0.2))\n",
    "model.add(Dense(500, activation='relu', input_shape=(n_cols, )))\n",
    "model.add(Dense(16, activation='softmax'))\n",
    "\n",
    "#Compile model\n",
    "model.compile(optimizer='sgd', loss='categorical_crossentropy', metrics=['accuracy'])\n",
    "\n",
    "#Run model with early stopping\n",
    "early_stopping_monitor = EarlyStopping(patience=2)\n",
    "model.fit(x_train, y_train, epochs=100, validation_data=(x_val, y_val), callbacks=[early_stopping_monitor])\n",
    "\n",
    "y_pred = model.predict(x_test)\n",
    "\n",
    "#For each of the testing data, find the type with highest probability\n",
    "pred_types = []\n",
    "types = y_train.columns\n",
    "for i in range(len(y_pred)):\n",
    "    pred_types.append(types[y_pred[i].argmax()])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(classification_report(y_test, pred_types))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Accuracy =\", accuracy_score(y_test, pred_types))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "from IPython.display import clear_output\n",
    "import re\n",
    "\n",
    "import nltk\n",
    "nltk.download('stopwords')\n",
    "from nltk.corpus import stopwords\n",
    "STOPWORDS = set(stopwords.words('english'))\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras.preprocessing.text import Tokenizer\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "from tensorflow.keras.callbacks import EarlyStopping\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import classification_report, confusion_matrix\n",
    "from imblearn.under_sampling import RandomUnderSampler\n",
    "\n",
    "import opendatasets as od"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#remove mbti-related text, so we dont classify a type based on how much some mbti string appears inside (not generalised)\n",
    "#for loop creates all stuff like XXTJ etc, cause some posts contains thing like this\n",
    "\n",
    "mbti_types = ['ISTJ', 'ISTP', 'ISFJ', 'ISFP', 'INTJ', 'INTP', 'INFJ', 'INFP', 'ESTJ', 'ESTP', 'ESFJ', 'ESFP', 'ENTJ', 'ENTP', 'ENFJ', 'ENFP']\n",
    "mbti_stopwords = ['fe', 'fi', 'ne', 'ni', 'se', 'si', 'te', 'ti']\n",
    "for i in mbti_types:\n",
    "  for mask in range(17):\n",
    "    tmp = list(i)\n",
    "    for j in range(4):\n",
    "      if pow(2, j) & mask != 0:\n",
    "        tmp[j] = 'X'\n",
    "    mbti_stopwords.append(\"\".join(tmp))\n",
    "    mbti_stopwords.append(\"\".join(tmp) + \"s\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}