TEXT CLASSIFICATION PROJECT.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#importing modules\n",
    "import os\n",
    "from nltk.corpus import stopwords\n",
    "import string\n",
    "import numpy as np\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.metrics import accuracy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#LIST OF FOLDER NAMES FROM WHICH FILES ARE TO BE EXTRACTED\n",
    "y=['alt.atheism',\n",
    " 'comp.graphics',\n",
    " 'comp.os.ms-windows.misc',\n",
    " 'comp.sys.ibm.pc.hardware',\n",
    " 'comp.sys.mac.hardware',\n",
    " 'comp.windows.x',\n",
    " 'misc.forsale',\n",
    " 'rec.autos',\n",
    " 'rec.motorcycles',\n",
    " 'rec.sport.baseball',\n",
    " 'rec.sport.hockey',\n",
    " 'sci.crypt',\n",
    " 'sci.electronics',\n",
    " 'sci.med',\n",
    " 'sci.space',\n",
    " 'soc.religion.christian',\n",
    " 'talk.politics.guns',\n",
    " 'talk.politics.mideast',\n",
    " 'talk.politics.misc',\n",
    " 'talk.religion.misc']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#MAKING OF STOP WORDS FROM MODULE NLTK\n",
    "stop=stopwords.words('english')\n",
    "punctuations=list(string.punctuation)\n",
    "stop = stop + punctuations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#path in my pc of the folder\n",
    "path='C:\\\\Users\\\\Psyfer\\\\Desktop\\\\ml\\\\20_newsgroups'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#INTIALIZING THE Y_TRUE (WHICH IS ACTUAL VALUES OF Y FOR COMPARING AND SCORES)\n",
    "y_true=[]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#MAKING OF VOCABUALARY AND THE Y_TRUE USING OS FUNCTIONS TO CHANGE DIRECTORY AND EXTRACT FILES FROM IT AS FILES ARE NOT\n",
    "#CURRENT DIRECTORY\n",
    "vocab={}\n",
    "for i in range(len(y)):\n",
    "    seq=os.listdir(path+s+y[i])\n",
    "    os.chdir(path+s+y[i])\n",
    "    for f in seq:\n",
    "        read=open(f)\n",
    "        y_true.append(i)\n",
    "        for l in read:\n",
    "            l=l.strip()\n",
    "            words=l.split()\n",
    "            for w in words:\n",
    "                if w.isalpha():\n",
    "                    if not w.lower() in stop:\n",
    "                        if w in vocab:\n",
    "                            vocab[w]+=1\n",
    "                        else:\n",
    "                            vocab[w]=1\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#IMPORTING MODULE COUNTER WHICH IS USED TO HAVE MOST FREQUENT WORDS\n",
    "from collections import Counter\n",
    "d=Counter(vocab)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#fEATURES ARE MOST COMMON 3000 WORDS FROM VOCAB\n",
    "f=d.most_common(3000)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#AS F IS LIST OF TUPLES AND WE WANT TO HAVE ONLY WORD THAT IS 0TH ENTRY OF EACH TUPLE IN LIST\n",
    "#THIS IS FINAL VOCAB\n",
    "v=[i[0] for i in f]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#INTIALIZING ARRAY TO HAVE WORD COUNT FOR EACH FEATURE IN EACH DOCUMENT\n",
    "r=[[0 for i in range(3000)] for j in range(len(y_true))]\n",
    "e=-1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#THIS IS CREATING A 2D ARRAY IN WHICH THE ENTRIES ARE THE WORD OR FEAURE COUNT IN EACH DOCUMENT\n",
    "for i in range(len(y)):\n",
    "    seq=os.listdir(path+s+y[i])\n",
    "    os.chdir(path+s+y[i])\n",
    "    \n",
    "    for f in seq:\n",
    "        e+=1\n",
    "        read=open(f)\n",
    "        for l in read:\n",
    "            l=l.strip()\n",
    "            words=l.split()\n",
    "            for w in words:\n",
    "                if w in v:\n",
    "                    r[e][v.index(w)]+=1\n",
    "    \n",
    "                "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#CREATING DATAFRAME FROM 2D ARRAY\n",
    "import pandas as pd\n",
    "df=pd.DataFrame(r)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#IMPORTING MULTINOMIAL NAIVE BAYES OF SKLEARN\n",
    "from sklearn.naive_bayes import MultinomialNB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#OBJECT OF MULTINOMIAL NAIVE BAYES\n",
    "clf = MultinomialNB()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#TRAINING DATA WITH DATAFRAME(df) AND Y_TRUE\n",
    "clf.fit(df, y_true)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#THIS IS TRAINING ACCURACY\n",
    "clf.score(df,y_true)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#CREATING LIST TO NUMPY ARRAY\n",
    "y_true=np.array(y_true)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#THIS IS FIT FUNCTION WHICH CREATES DICTIONARY WHICH WILL BE USED TO PREDICT DATA\n",
    "def fit(x_train,y_train):\n",
    "    count={}\n",
    "    possible_classes=list(set(y_train))\n",
    "    count['total_data']=len(y_train)\n",
    "    for current_class in possible_classes:\n",
    "        x=x_train[y_train==current_class]\n",
    "        count[current_class]={}\n",
    "        count[current_class]['total_count']=len(x)\n",
    "        l=x.sum()\n",
    "        l=np.array(l)\n",
    "        \n",
    "        count[current_class]['total_words']=l.sum()\n",
    "\n",
    "        for f in range(3000):\n",
    "            count[current_class][f]=x[:][f].sum()\n",
    "    return count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#THIS RETURN PROBABILTY OF EACH CLASS FOR A DOCUMENT WHICH HAS WORD BANK AS X_TEST_VOCAB\n",
    "def probability_singler(count,x_test_vocab,current_class):\n",
    "    prob=count[current_class]['total_count']/count['total_data']\n",
    "    for w in x_test_vocab:\n",
    "        if w in v:\n",
    "            we=v.index(w)\n",
    "            we=(count[current_class][we]+1)/(count[current_class]['total_words'] + len(v) )\n",
    "            prob=(prob)*(we)\n",
    "    return prob\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#THIS RETURN BEST CLASS FOR A DOCUMENT BY COMPARING PROBABILITY AND INPUT X_TEST_VOCAB WHICH IS WORD BANK IN TEST\n",
    "#DOCUMENT\n",
    "def probability(count,x_test_vocab,y_train):\n",
    "    best_p=-10000\n",
    "    best_class=-1\n",
    "    possible_classes=list(set(y_train))\n",
    "    for current_class in possible_classes:\n",
    "        prob=probability_singler(count,x_test_vocab,current_class)\n",
    "        \n",
    "        if(prob>best_p):\n",
    "            best_p=prob\n",
    "            best_class=current_class\n",
    "    return best_class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#DICTIONARY COUNT OF FIT FUNCTION\n",
    "count=fit(df,y_true)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#THIS IS THE NAMES OF FOLDER CONTAINING RANDOM TEST FILES FROM EACH CLASS\n",
    "y_test_folder=['test'+str(i) for i in range(20)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#THIS IS PATH TO TEST FILES\n",
    "path='C:\\\\Users\\\\Psyfer\\\\Desktop\\\\ml\\\\20_newsgroups'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#INTIALISING y_predict WHICH IS PREDICTED BY OWN CODE AND Y_TEST_TRUE IS TRUE VALUES OF PREDICTION(AS TESTING FILES ARE\n",
    "#MADE SUCH WE CAN HAVE TRUE VALUES OF PREDICTION FOR TESTING FILES AND COMPARE RESULT\n",
    "y_predict=[]\n",
    "y_test_true=[]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#INTIALSING 2D ARRAY\n",
    "e=-1\n",
    "we=[[0 for i in range(3000)] for j in range(2799)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#CREATING Y_PREDICT AND Y_TEST_TRUE FOR TESTING FILES\n",
    "for i in range(len(y_test_folder)):\n",
    "    #y_test=[]\n",
    "    seq=os.listdir(path+s+y_test_folder[i])\n",
    "    os.chdir(path+s+y_test_folder[i])\n",
    "    for f in seq:\n",
    "            e+=1\n",
    "            l=[]\n",
    "            read=open(f)\n",
    "            for line in read:\n",
    "                line=line.strip()\n",
    "                words=line.split()\n",
    "                for w in words:\n",
    "                    if w.isalpha():\n",
    "                        if not w.lower() in stop:\n",
    "                            if w in v:\n",
    "                                we[e][v.index(w)]+=1\n",
    "                                \n",
    "                                l.append(w)\n",
    "            y_predict.append(probability(count,l,y_true))\n",
    "            \n",
    "    \n",
    "            y_test_true.append(i)\n",
    "   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#CREATING 2D ARRAY WHICH IS USED BY SKLEARN NAIVE BAYES TO PREDICT\n",
    "rt=pd.DataFrame(we)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#PREDICTING on test files USING SKLEARN NAIVE BAYES \n",
    "y_predict_sk=clf.predict(rt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#TESTING ACCURACY\n",
    "clf.score(rt,y_test_true)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#CLASSIFICATION REPORT FOR TESTING FILES BY OWN CODE\n",
    "print(classification_report(y_test_true, y_predict))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#CLASSIFICATION REPORT FOR TESTING FILES BY SKLAEARN naive bayes\n",
    "print(classification_report(y_test_true, y_predict_sk))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}