Disaster_Tweets_classification-LSTM_Model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "7898baff",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Dataset: https://www.kaggle.com/competitions/nlp-getting-started/overview"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "1b6737b5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>keyword</th>\n",
       "      <th>location</th>\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Our Deeds are the Reason of this #earthquake M...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Forest fire near La Ronge Sask. Canada</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>All residents asked to 'shelter in place' are ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>13,000 people receive #wildfires evacuation or...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Just got sent this photo from Ruby #Alaska as ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id keyword location                                               text  \\\n",
       "0   1     NaN      NaN  Our Deeds are the Reason of this #earthquake M...   \n",
       "1   4     NaN      NaN             Forest fire near La Ronge Sask. Canada   \n",
       "2   5     NaN      NaN  All residents asked to 'shelter in place' are ...   \n",
       "3   6     NaN      NaN  13,000 people receive #wildfires evacuation or...   \n",
       "4   7     NaN      NaN  Just got sent this photo from Ruby #Alaska as ...   \n",
       "\n",
       "   target  \n",
       "0       1  \n",
       "1       1  \n",
       "2       1  \n",
       "3       1  \n",
       "4       1  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df = pd.read_csv(\"train.csv\")\n",
    "test_df = pd.read_csv(\"test.csv\")\n",
    "\n",
    "train_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Checking for NaN values in each column"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id             0\n",
       "keyword       61\n",
       "location    2533\n",
       "text           0\n",
       "target         0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id             0\n",
       "keyword       26\n",
       "location    1105\n",
       "text           0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df.isna().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " In this code we will only consider keywords and target column. Both the columns do not have any NA values."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Removing duplicates "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7613, 2)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df = train_df[['text','target']]\n",
    "train_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7521, 2)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df = train_df.drop_duplicates()\n",
    "train_df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Analysis on Target column"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "a38e40bf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 0])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df['target'].unique()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "13fe3999",
   "metadata": {},
   "source": [
    "#### Text cleaning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "e37278f3",
   "metadata": {},
   "outputs": [],
   "source": [
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "3fc152d5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'What a goooooooaaaaaal!!!!!!'"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df['text'][19]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8a6a817f",
   "metadata": {},
   "source": [
    "Steps To be taken to clean the text\n",
    "- Remove punctutaions #.,-?! from text containing it. \n",
    "- Remove numerics\n",
    "- convert text to lower case"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "8cf5a2b4",
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_data(text):\n",
    "    \n",
    "    for i in range(0, len(text)):\n",
    "        \n",
    "        # Remove urls\n",
    "        text[i] = re.sub('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\\(\\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+',\"\",text[i])\n",
    "        \n",
    "        # Remove Hashtags\n",
    "        text[i] = re.sub('#(\\w+)',\"\",text[i])\n",
    "        \n",
    "        # Remove @tags\n",
    "        text[i] = re.sub('@(\\w+)',\"\",text[i])\n",
    "        \n",
    "        # Remove &tags\n",
    "        text[i] = re.sub('&(\\w+)',\"\",text[i])\n",
    "        \n",
    "        # Remove Non ASCII characters\n",
    "        text[i] = re.sub(r'[^\\x00-\\x7F]+',' ', text[i])\n",
    "        \n",
    "        # Remove numbers [0-9]\n",
    "        text[i] = re.sub('[0-9]',\"\",text[i])\n",
    "        \n",
    "        # Split words seprated with -\n",
    "        text[i] = re.sub('-',\" \",text[i])\n",
    "        \n",
    "        text[i] = re.sub('[#.,;?!)/^(}{:%+=$*\\|~_]',' ',text[i])\n",
    "        \n",
    "        # Remove square brackets\n",
    "        text[i] = re.sub('[\\[\\]]',' ',text[i])\n",
    "        \n",
    "        # Remove \\n from text\n",
    "        text[i] = re.sub('\\n',' ',text[i])\n",
    "        \n",
    "        text[i] = re.sub('[\\']',' ',text[i])\n",
    "        \n",
    "        # Remove extra space\n",
    "        text[i] = re.sub(' +', ' ', text[i])\n",
    "        \n",
    "        \n",
    "    return text"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras.preprocessing.text import Tokenizer\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "import numpy as np\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# shuffle data of training set\n",
    "\n",
    "train_df = train_df.sample(frac = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "3d5c0d4a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Clean training data\n",
    "\n",
    "data_x = train_df['text'].tolist()\n",
    "data_y = train_df['target'].tolist()\n",
    "\n",
    "data_x = clean_data(data_x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7521\n",
      "7521\n"
     ]
    }
   ],
   "source": [
    "print(len(data_x))\n",
    "print(len(data_y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# split the data into training and validation sets\n",
    "\n",
    "train_len = int(0.8 * len(data_x))\n",
    "\n",
    "train_x = np.array(data_x[0:train_len])\n",
    "train_y = np.array(data_y[0:train_len])\n",
    "\n",
    "val_x = np.array(data_x[train_len:])\n",
    "val_y = np.array(data_y[train_len:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Length of training data  6016 6016\n",
      "Length of validation data 1505 1505\n"
     ]
    }
   ],
   "source": [
    "print(\"Length of training data \", len(train_x), len(train_y))\n",
    "print(\"Length of validation data\", len(val_x), len(val_y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Vocabulary Size\n",
    "\n",
    "voc_size = 30000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Training a tokenizer\n",
    "\n",
    "#tokenizer = Tokenizer(num_words=voc_size,filters='!\"#$%&()*+,-./:;<=>?@[\\\\]^_`{|}~\\t\\n',lower=True, split=' ')\n",
    "tokenizer = Tokenizer(num_words=voc_size,filters=\"\",lower=True, split=' ')\n",
    "tokenizer.fit_on_texts(train_x)\n",
    "word_index = tokenizer.word_index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(6016, 33)\n",
      "33\n"
     ]
    }
   ],
   "source": [
    "# Padding Sequences so that the length of each sequence is same\n",
    "# Setting maxlen = None, so that sequences will be padded to the length of the longest individual sequence\n",
    "\n",
    "train_seq = tokenizer.texts_to_sequences(train_x)\n",
    "train_pad_seq = pad_sequences(train_seq, padding='post', maxlen=None)\n",
    "\n",
    "MAX_SEQUENCE_LENGTH = train_pad_seq.shape[1]\n",
    "\n",
    "print(train_pad_seq.shape)\n",
    "print(MAX_SEQUENCE_LENGTH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1505, 33)\n"
     ]
    }
   ],
   "source": [
    "# Converting val data to sequences\n",
    "\n",
    "val_seq = tokenizer.texts_to_sequences(val_x)\n",
    "val_pad_seq = pad_sequences(val_seq, padding='post', maxlen=MAX_SEQUENCE_LENGTH)\n",
    "\n",
    "print(val_pad_seq.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11386\n"
     ]
    }
   ],
   "source": [
    "print(len(tokenizer.word_index))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "voc_size = len(tokenizer.word_index)+1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### SIMPLE MODEL CREATION"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras.layers import Embedding,Bidirectional, Input, LSTM, Dropout, concatenate\n",
    "from tensorflow.keras.layers import Dense, Flatten, Conv2D, MaxPool2D, Lambda, Reshape\n",
    "from tensorflow.keras import Model, Sequential\n",
    "from tensorflow.keras.utils import plot_model\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding (Embedding)        (None, 33, 300)           3416100   \n",
      "_________________________________________________________________\n",
      "lstm (LSTM)                  (None, 100)               160400    \n",
      "_________________________________________________________________\n",
      "flatten (Flatten)            (None, 100)               0         \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 1)                 101       \n",
      "=================================================================\n",
      "Total params: 3,576,601\n",
      "Trainable params: 3,576,601\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-05-13 23:26:27.429630: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  SSE4.1 SSE4.2 AVX AVX2 AVX512F FMA\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2022-05-13 23:26:27.431448: I tensorflow/core/common_runtime/process_util.cc:146] Creating new thread pool with default inter op setting: 2. Tune using inter_op_parallelism_threads for best performance.\n"
     ]
    }
   ],
   "source": [
    "EMBEDDING_DIM = 300\n",
    "\n",
    "model = Sequential()\n",
    "model.add(Embedding(input_dim = voc_size,output_dim = EMBEDDING_DIM, input_length=MAX_SEQUENCE_LENGTH))\n",
    "model.add(LSTM(100))\n",
    "model.add(Flatten())\n",
    "model.add(Dense(1, activation= 'sigmoid'))\n",
    "\n",
    "# compile the model\n",
    "model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])\n",
    "\n",
    "# Model summary\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-05-13 23:26:34.439572: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:185] None of the MLIR Optimization Passes are enabled (registered 2)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "188/188 [==============================] - 20s 101ms/step - loss: 0.6072 - accuracy: 0.6662 - val_loss: 0.5061 - val_accuracy: 0.7920\n",
      "Epoch 2/10\n",
      "188/188 [==============================] - 14s 75ms/step - loss: 0.3889 - accuracy: 0.8444 - val_loss: 0.5014 - val_accuracy: 0.7960\n",
      "Epoch 3/10\n",
      "188/188 [==============================] - 13s 69ms/step - loss: 0.2419 - accuracy: 0.9141 - val_loss: 0.5285 - val_accuracy: 0.7754\n",
      "Epoch 4/10\n",
      "188/188 [==============================] - 15s 81ms/step - loss: 0.1652 - accuracy: 0.9496 - val_loss: 0.6308 - val_accuracy: 0.7635\n",
      "Epoch 5/10\n",
      "188/188 [==============================] - 17s 87ms/step - loss: 0.1238 - accuracy: 0.9606 - val_loss: 0.8047 - val_accuracy: 0.7641\n",
      "Epoch 6/10\n",
      "188/188 [==============================] - 15s 80ms/step - loss: 0.1159 - accuracy: 0.9608 - val_loss: 1.0537 - val_accuracy: 0.7654\n",
      "Epoch 7/10\n",
      "188/188 [==============================] - 16s 84ms/step - loss: 0.1025 - accuracy: 0.9641 - val_loss: 0.9395 - val_accuracy: 0.7648\n",
      "Epoch 8/10\n",
      "188/188 [==============================] - 16s 84ms/step - loss: 0.0877 - accuracy: 0.9661 - val_loss: 0.9119 - val_accuracy: 0.7654\n",
      "Epoch 9/10\n",
      "188/188 [==============================] - 19s 103ms/step - loss: 0.0654 - accuracy: 0.9754 - val_loss: 1.0850 - val_accuracy: 0.7635\n",
      "Epoch 10/10\n",
      "188/188 [==============================] - 13s 71ms/step - loss: 0.0605 - accuracy: 0.9756 - val_loss: 1.0578 - val_accuracy: 0.7714\n"
     ]
    }
   ],
   "source": [
    "# fit the model\n",
    "\n",
    "history = model.fit(x=train_pad_seq, y=train_y, epochs=10, validation_data=(val_pad_seq,val_y),workers=4, use_multiprocessing=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Use of Google Word2Vec Embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "dc97f199",
   "metadata": {},
   "outputs": [],
   "source": [
    "from gensim.models import keyedvectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "7460e3ef",
   "metadata": {},
   "outputs": [],
   "source": [
    "filename = 'GoogleNews-vectors-negative300.bin'\n",
    "google_embeddings_model = keyedvectors.load_word2vec_format(filename, binary=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "0b6f1407",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2453"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# The index of vector for a token which is present in the vocabulary\n",
    "\n",
    "rock_idx = google_embeddings_model.key_to_index[\"rock\"]\n",
    "rock_idx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "bee3f361",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(300,)\n",
      "(300,)\n"
     ]
    }
   ],
   "source": [
    "# The embedded vector for specific token which is present in the vocabulary\n",
    "\n",
    "vector1 = google_embeddings_model.get_vector(\"goal\",norm=True)\n",
    "vector2 = google_embeddings_model.get_vector(\"goal\")\n",
    "\n",
    "print(vector1.shape)\n",
    "print(vector2.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "b313c987",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_weight_matrix(model, vocab):\n",
    "    \n",
    "    # total vocabulary size plus 0 for unknown words\n",
    "    vocab_size = len(vocab) + 1\n",
    "    weight_matrix = np.zeros((vocab_size, 300))\n",
    "    \n",
    "    for word, i in vocab.items():\n",
    "        try:\n",
    "            weight_matrix[i] = model.get_vector(word,norm=True)\n",
    "        except:\n",
    "            weight_matrix[i] = np.random.uniform(low=-0.1, high=0.1, size=300)\n",
    "            \n",
    "    return weight_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_vectors = get_weight_matrix(google_embeddings_model,tokenizer.word_index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_1\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_1 (Embedding)      (None, 33, 300)           3416100   \n",
      "_________________________________________________________________\n",
      "lstm_1 (LSTM)                (None, 100)               160400    \n",
      "_________________________________________________________________\n",
      "flatten_1 (Flatten)          (None, 100)               0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 1)                 101       \n",
      "=================================================================\n",
      "Total params: 3,576,601\n",
      "Trainable params: 160,501\n",
      "Non-trainable params: 3,416,100\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "EMBEDDING_DIM =300\n",
    "\n",
    "model1 = Sequential()\n",
    "model1.add(Embedding(input_dim = voc_size,output_dim = EMBEDDING_DIM, weights=[embedding_vectors], input_length=MAX_SEQUENCE_LENGTH, trainable=False))\n",
    "model1.add(LSTM(100))\n",
    "model1.add(Flatten())\n",
    "model1.add(Dense(1, activation= 'sigmoid'))\n",
    "\n",
    "# compile the model\n",
    "model1.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])\n",
    "\n",
    "# Model summary\n",
    "model1.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "188/188 [==============================] - 10s 49ms/step - loss: 0.5238 - accuracy: 0.7463 - val_loss: 0.5137 - val_accuracy: 0.7721\n",
      "Epoch 2/20\n",
      "188/188 [==============================] - 11s 61ms/step - loss: 0.4409 - accuracy: 0.8034 - val_loss: 0.4529 - val_accuracy: 0.8020\n",
      "Epoch 3/20\n",
      "188/188 [==============================] - 13s 67ms/step - loss: 0.4328 - accuracy: 0.8108 - val_loss: 0.4715 - val_accuracy: 0.7880\n",
      "Epoch 4/20\n",
      "188/188 [==============================] - 25s 133ms/step - loss: 0.4207 - accuracy: 0.8182 - val_loss: 0.4609 - val_accuracy: 0.8007\n",
      "Epoch 5/20\n",
      "188/188 [==============================] - 11s 56ms/step - loss: 0.4144 - accuracy: 0.8255 - val_loss: 0.4700 - val_accuracy: 0.8053\n",
      "Epoch 6/20\n",
      "188/188 [==============================] - 13s 69ms/step - loss: 0.4057 - accuracy: 0.8250 - val_loss: 0.5248 - val_accuracy: 0.7900\n",
      "Epoch 7/20\n",
      "188/188 [==============================] - 14s 74ms/step - loss: 0.4043 - accuracy: 0.8296 - val_loss: 0.4462 - val_accuracy: 0.8047\n",
      "Epoch 8/20\n",
      "188/188 [==============================] - 16s 88ms/step - loss: 0.3947 - accuracy: 0.8319 - val_loss: 0.4594 - val_accuracy: 0.8020\n",
      "Epoch 9/20\n",
      "188/188 [==============================] - 16s 86ms/step - loss: 0.3929 - accuracy: 0.8353 - val_loss: 0.4746 - val_accuracy: 0.8060\n",
      "Epoch 10/20\n",
      "188/188 [==============================] - 11s 59ms/step - loss: 0.3879 - accuracy: 0.8399 - val_loss: 0.4583 - val_accuracy: 0.7967\n",
      "Epoch 11/20\n",
      "188/188 [==============================] - 14s 75ms/step - loss: 0.3726 - accuracy: 0.8411 - val_loss: 0.4723 - val_accuracy: 0.8040\n",
      "Epoch 12/20\n",
      "188/188 [==============================] - 13s 69ms/step - loss: 0.3688 - accuracy: 0.8446 - val_loss: 0.4578 - val_accuracy: 0.8027\n",
      "Epoch 13/20\n",
      "188/188 [==============================] - 12s 66ms/step - loss: 0.3610 - accuracy: 0.8501 - val_loss: 0.4987 - val_accuracy: 0.8020\n",
      "Epoch 14/20\n",
      "188/188 [==============================] - 12s 63ms/step - loss: 0.3578 - accuracy: 0.8499 - val_loss: 0.4812 - val_accuracy: 0.7847\n",
      "Epoch 15/20\n",
      "188/188 [==============================] - 10s 53ms/step - loss: 0.3485 - accuracy: 0.8526 - val_loss: 0.5159 - val_accuracy: 0.7641\n",
      "Epoch 16/20\n",
      "188/188 [==============================] - 12s 65ms/step - loss: 0.3386 - accuracy: 0.8610 - val_loss: 0.4904 - val_accuracy: 0.7940\n",
      "Epoch 17/20\n",
      "188/188 [==============================] - 13s 70ms/step - loss: 0.3245 - accuracy: 0.8667 - val_loss: 0.5047 - val_accuracy: 0.7927\n",
      "Epoch 18/20\n",
      "188/188 [==============================] - 13s 70ms/step - loss: 0.3248 - accuracy: 0.8705 - val_loss: 0.4746 - val_accuracy: 0.8000\n",
      "Epoch 19/20\n",
      "188/188 [==============================] - 12s 63ms/step - loss: 0.3097 - accuracy: 0.8745 - val_loss: 0.5307 - val_accuracy: 0.7973\n",
      "Epoch 20/20\n",
      "188/188 [==============================] - 12s 63ms/step - loss: 0.3032 - accuracy: 0.8793 - val_loss: 0.5611 - val_accuracy: 0.7867\n"
     ]
    }
   ],
   "source": [
    "# fit the model\n",
    "\n",
    "history = model1.fit(x=train_pad_seq, y=train_y, epochs=20, validation_data=(val_pad_seq,val_y),workers=4, use_multiprocessing=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Use of Glove Embeddings "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "from gensim.scripts.glove2word2vec import glove2word2vec\n",
    "from gensim.models import keyedvectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_46457/389290134.py:5: DeprecationWarning: Call to deprecated `glove2word2vec` (KeyedVectors.load_word2vec_format(.., binary=False, no_header=True) loads GLoVE text vectors.).\n",
      "  glove2word2vec(glove_input_file, word2vec_output_file)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(400000, 100)"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# convert the GloVe file format to the Word2Vec file format. Once converted, the file can be loaded just like Word2Vec\n",
    "\n",
    "glove_input_file = 'glove.6B.100d.txt'\n",
    "word2vec_output_file = 'glove.6B.100d.txt.word2vec'\n",
    "glove2word2vec(glove_input_file, word2vec_output_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load Glove Embeddings\n",
    "\n",
    "filename = 'glove.6B.100d.txt.word2vec'\n",
    "glove_embeddings_model = keyedvectors.load_word2vec_format(filename, binary=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(100,)\n"
     ]
    }
   ],
   "source": [
    "# The embedded vector for specific token which is present in the vocabulary\n",
    "\n",
    "vector1 = glove_embeddings_model.get_vector(\"goal\")\n",
    "\n",
    "print(vector1.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_weight_matrix(model, vocab):\n",
    "    \n",
    "    # total vocabulary size plus 0 for unknown words\n",
    "    vocab_size = len(vocab) + 1\n",
    "    weight_matrix = np.zeros((vocab_size, 100))\n",
    "    \n",
    "    for word, i in vocab.items():\n",
    "        try:\n",
    "            weight_matrix[i] = model.get_vector(word, norm=True)\n",
    "        except:\n",
    "            weight_matrix[i] = np.random.uniform(low=-0.1, high=0.1, size=100)\n",
    "            \n",
    "    return weight_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_vectors = get_weight_matrix(glove_embeddings_model,tokenizer.word_index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_2\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_2 (Embedding)      (None, 33, 100)           1138700   \n",
      "_________________________________________________________________\n",
      "lstm_2 (LSTM)                (None, 100)               80400     \n",
      "_________________________________________________________________\n",
      "flatten_2 (Flatten)          (None, 100)               0         \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 1)                 101       \n",
      "=================================================================\n",
      "Total params: 1,219,201\n",
      "Trainable params: 80,501\n",
      "Non-trainable params: 1,138,700\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "EMBEDDING_DIM =100\n",
    "\n",
    "model1 = Sequential()\n",
    "model1.add(Embedding(input_dim = voc_size,output_dim = EMBEDDING_DIM, weights=[embedding_vectors], input_length=MAX_SEQUENCE_LENGTH, trainable=False))\n",
    "model1.add(LSTM(100))\n",
    "model1.add(Flatten())\n",
    "model1.add(Dense(1, activation= 'sigmoid'))\n",
    "\n",
    "# compile the model\n",
    "model1.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])\n",
    "\n",
    "# Model summary\n",
    "model1.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "188/188 [==============================] - 11s 56ms/step - loss: 0.5133 - accuracy: 0.7488 - val_loss: 0.4709 - val_accuracy: 0.7801\n",
      "Epoch 2/10\n",
      "188/188 [==============================] - 9s 49ms/step - loss: 0.4511 - accuracy: 0.7989 - val_loss: 0.4706 - val_accuracy: 0.7741\n",
      "Epoch 3/10\n",
      "188/188 [==============================] - 8s 43ms/step - loss: 0.4449 - accuracy: 0.7997 - val_loss: 0.4611 - val_accuracy: 0.7980\n",
      "Epoch 4/10\n",
      "188/188 [==============================] - 7s 37ms/step - loss: 0.4425 - accuracy: 0.8085 - val_loss: 0.4596 - val_accuracy: 0.7987\n",
      "Epoch 5/10\n",
      "188/188 [==============================] - 10s 51ms/step - loss: 0.4376 - accuracy: 0.8024 - val_loss: 0.4507 - val_accuracy: 0.8047\n",
      "Epoch 6/10\n",
      "188/188 [==============================] - 9s 49ms/step - loss: 0.4316 - accuracy: 0.8092 - val_loss: 0.4655 - val_accuracy: 0.8027\n",
      "Epoch 7/10\n",
      "188/188 [==============================] - 11s 60ms/step - loss: 0.4319 - accuracy: 0.8075 - val_loss: 0.4521 - val_accuracy: 0.7967\n",
      "Epoch 8/10\n",
      "188/188 [==============================] - 7s 38ms/step - loss: 0.4307 - accuracy: 0.8098 - val_loss: 0.4551 - val_accuracy: 0.7907\n",
      "Epoch 9/10\n",
      "188/188 [==============================] - 8s 42ms/step - loss: 0.4204 - accuracy: 0.8157 - val_loss: 0.4476 - val_accuracy: 0.8086\n",
      "Epoch 10/10\n",
      "188/188 [==============================] - 9s 49ms/step - loss: 0.4206 - accuracy: 0.8140 - val_loss: 0.4619 - val_accuracy: 0.8027\n"
     ]
    }
   ],
   "source": [
    "# fit the model\n",
    "\n",
    "history = model1.fit(x=train_pad_seq, y=train_y, epochs=10, validation_data=(val_pad_seq,val_y),workers=4, use_multiprocessing=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}