Step 3- Feature_extration_model_training.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "feature_extration_model_training.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Cj0uxaDyXPoY",
        "colab_type": "text"
      },
      "source": [
        "# **Feature Extraction Models**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xm0lbbfSFw_n",
        "colab_type": "code",
        "outputId": "d5b1c6ff-83e8-4edf-ba26-c080b5327ba2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 83
        }
      },
      "source": [
        "import pandas as pd\n",
        "import nltk\n",
        "import gensim\n",
        "from nltk.stem import WordNetLemmatizer, SnowballStemmer\n",
        "from gensim.parsing.preprocessing import STOPWORDS\n",
        "import numpy as np\n",
        "from gensim import corpora,models\n",
        "import time\n",
        "import pickle\n",
        "nltk.download('stopwords')\n",
        "nltk.download('wordnet')\n",
        "import warnings\n",
        "warnings.filterwarnings(\"ignore\", category=DeprecationWarning)"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/stopwords.zip.\n",
            "[nltk_data] Downloading package wordnet to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/wordnet.zip.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "c-hMZcibFzc3",
        "colab_type": "code",
        "outputId": "6940d2ab-5835-4a9b-f63e-0357ab8b7500",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 120
        }
      },
      "source": [
        "#Mount google drive\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hyhp-dKUHEed",
        "colab_type": "code",
        "outputId": "6ac21a34-1db1-4aef-96ed-95c0c24299d3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 70
        }
      },
      "source": [
        "# open a file, where you stored the pickled data\n",
        "f = open('drive/My Drive/duplicate_detection/bow_corpus.pickle', 'rb')\n",
        "bow_corpus=pickle.load(f)\n",
        "\n",
        "file = open('drive/My Drive/duplicate_detection/dictionary.pickle', 'rb')\n",
        "dictionary=pickle.load(file)\n",
        "\n",
        "# later on, load trained model from file\n",
        "lda_model =  models.LdaModel.load('drive/My Drive/duplicate_detection/lda_model.model')"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/smart_open/smart_open_lib.py:253: UserWarning: This function is deprecated, use smart_open.open instead. See the migration notes for details: https://github.com/RaRe-Technologies/smart_open/blob/master/README.rst#migrating-to-the-new-open-function\n",
            "  'See the migration notes for details: %s' % _MIGRATION_NOTES_URL\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9Dm8jFprLADw",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def lemmatize(text):\n",
        "    return WordNetLemmatizer().lemmatize(text, pos='v')\n",
        "\n",
        "def preprocess(text):\n",
        "    result = []\n",
        "    for token in gensim.utils.simple_preprocess(text):\n",
        "        if token not in gensim.parsing.preprocessing.STOPWORDS and len(token) > 5:\n",
        "            result.append(lemmatize(token))\n",
        "    return result"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wpCTWfbzK8sa",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Import all the clusters from the drive \n",
        "for c in range(10):\n",
        "  exec('topic_{} = pd.read_csv(\"drive/My Drive/Mozilla/topic_{}.csv\")'.format(c,c))\n",
        "  exec(\"topic_{}= topic_{}.drop(columns=['Unnamed: 0'])\".format(c,c))\n",
        "  exec(\"topic_{}['Description'] = topic_{}['Description'].map(preprocess)\".format(c,c))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oiZ93ze1VI6_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#creating a corpus for Word2Vec and FastText models\n",
        "for i in range(10):\n",
        "  exec('sent_{} = []'.format(i))\n",
        "  exec('x= topic_{}'.format(i))\n",
        "  for j in range(len(x)):\n",
        "    exec('sent_{}.append(topic_{}.Description[{}])'.format(i,i,j))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yyBtELSBhZkw",
        "colab_type": "code",
        "outputId": "6a439bb7-80fb-487a-dd17-3be924f4f6b0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 183
        }
      },
      "source": [
        "#Length of all the corpus\n",
        "for sent in range(10):\n",
        "  exec('print(len(sent_{}))'.format(sent))"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "10751\n",
            "673\n",
            "5619\n",
            "8009\n",
            "11362\n",
            "3037\n",
            "6982\n",
            "8172\n",
            "12091\n",
            "2831\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7MN3B8FMcFRJ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# !pip install glove_python"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aaAVwfqHVM3I",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from gensim.models import Word2Vec,FastText\n",
        "from gensim.test.utils import get_tmpfile\n",
        "from glove import Glove, Corpus"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "he0V2e8PYFpy",
        "colab_type": "text"
      },
      "source": [
        "### **Word2Vec model training**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "n82GYprkVU_U",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "for cluster in range(10):\n",
        "  #Preparing parameters for model training\n",
        "  exec('corpus = sent_{}'.format(cluster))\n",
        "  vector_size = 100\n",
        "  w = 6\n",
        "  min_count = 5\n",
        "  CBoW = 0\n",
        "  iterations = 100\n",
        "\n",
        "  #Training Word2Vec model for each cluster\n",
        "  exec ('w2vmodel{} = Word2Vec(corpus, size=vector_size, window=w, min_count=min_count, sg=CBoW, iter=iterations)'.format(cluster))\n",
        "\n",
        "  #Save the all the models in individual file\n",
        "  exec('path = get_tmpfile(\"drive/My Drive/duplicate_detection/word2vec{}.model\")'.format(cluster))\n",
        "  exec('w2vmodel{}.save(\"drive/My Drive/duplicate_detection/word2vec{}.model\")'.format(cluster), cluster))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dqxoHhzCZIVi",
        "colab_type": "text"
      },
      "source": [
        "### **FastText model training**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_g6HDHEylpRh",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 70
        },
        "outputId": "33629426-000f-476f-e4b8-4b60dc34a747"
      },
      "source": [
        "for cluster in range(10):\n",
        "  #Preparing parameters for model training\n",
        "  exec('corpus = sent_{}'.format(cluster))\n",
        "  vector_size = 100\n",
        "  w = 6\n",
        "  min_count = 5\n",
        "  CBoW = 0\n",
        "  iterations = 100\n",
        "\n",
        "  #Training FastText model for each cluster\n",
        "  exec ('ftmodel{} = FastText(corpus, size=vector_size, window=w, min_count=min_count, sg=CBoW, iter=iterations)'.format(cluster))\n",
        "\n",
        "  #Save the all the models in individual file\n",
        "  exec('path = get_tmpfile(\"drive/My Drive/duplicate_detection/ftmodel{}.model\")'.format(cluster))\n",
        "  exec('ftmodel{}.save(\"drive/My Drive/duplicate_detection/ftmodel{}.model\")'.format(cluster, cluster))"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/smart_open/smart_open_lib.py:253: UserWarning: This function is deprecated, use smart_open.open instead. See the migration notes for details: https://github.com/RaRe-Technologies/smart_open/blob/master/README.rst#migrating-to-the-new-open-function\n",
            "  'See the migration notes for details: %s' % _MIGRATION_NOTES_URL\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kCdnHdsUc3Dm",
        "colab_type": "text"
      },
      "source": [
        "### **GloVe model training**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pjqrrgitqHgX",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Training GloVe model for each cluster\n",
        "for cluster in range(10):\n",
        "  vector_size = 100\n",
        "  exec('glove_corpus{}=Corpus()'.format(cluster, cluster)) \n",
        "  exec('glove_corpus{}.fit(sent_{})'.format(cluster, cluster))\n",
        "  exec('glove{}= Glove(no_components=vector_size, learning_rate=0.18, alpha=0.75, max_count=100, max_loss=10.0, random_state=None)'.format(cluster, cluster))\n",
        "  exec('glove{}.fit(glove_corpus{}.matrix, epochs=1000, no_threads=3, verbose=True)'.format(cluster, cluster))\n",
        "  exec('transformer = lambda dictionary2:glove{}.transform_paragraph(words, epochs=1000,ignore_missing=False)'.format(cluster, cluster))\n",
        "  exec('glove{}.add_dictionary(glove_corpus{}.dictionary)'.format(cluster, cluster))\n",
        "\n",
        "  #Save the all the models in individual file\n",
        "  exec('path = get_tmpfile(\"drive/My Drive/duplicate_detection/glove{}.model\")'.format(cluster))\n",
        "  exec('glove{}.save(\"drive/My Drive/duplicate_detection/glove{}.model\")'.format(cluster, cluster))"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}