Inverted Index.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "bdaba149-1843-447f-aec3-2b67dd216709",
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk\n",
    "import csv\n",
    "import math\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import warnings\n",
    "from nltk.tokenize import word_tokenize, wordpunct_tokenize, sent_tokenize\n",
    "from nltk.corpus import stopwords\n",
    "from collections import Counter"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8b4be9b1-eb01-4c83-89f3-da76fd71f1cb",
   "metadata": {},
   "source": [
    "## 1. Position Indices"
   ]
  },
  {
   "cell_type": "raw",
   "id": "a31a539b-ef34-47a4-b2b9-1f27ece526b5",
   "metadata": {},
   "source": [
    "#term: doc1: <position1, position2, ...>; doc2: <position1, position2, ...>\n",
    "angels: 2: <36,174,252,651>; 4: <12,22,102,432>; 7: <17>;\n",
    "fools: 2: <1,17,74,222>; 4: <8,78,108,458>; 7: <3,13,23,193>;\n",
    "fear: 2: <87,704,722,901>; 4: <13,43,113,433>; 7: <18,328,528>;\n",
    "in: 2: <3,37,76,444,851>; 4: <10,20,110,470,500>; 7: <5,15,25,195>;\n",
    "rush: 2: <2,66,194,321,702>; 4: <9,69,149,429,569>; 7: <4,14,404>;\n",
    "to: 2: <47,86,234,999>; 4: <14,24,774,944>; 7: <199,319,599,709>;\n",
    "tread: 2: <57,94,333>; 4: <15,35,155>; 7: <20,320>;\n",
    "where: 2: <67,124,393,1001>; 4: <11,41,101,421,431>; 7: <15,35,735>;"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0711210b-924c-49f5-a196-85c28beea8fb",
   "metadata": {},
   "source": [
    "### Read file.\n",
    "#### Store positional index as a text file. \n",
    "#### Store as a nested dictionary. With postings as values (dictionary type) and terms as keys."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "9e632870-1f62-4c3d-a80e-a46e6b86d4d1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Positional Index data:\n",
      "{'angels': {'2': [36, 174, 252, 651], '4': [12, 22, 102, 432], '7': [17]}, 'fools': {'2': [1, 17, 74, 222], '4': [8, 78, 108, 458], '7': [3, 13, 23, 193]}, 'fear': {'2': [87, 704, 722, 901], '4': [13, 43, 113, 433], '7': [18, 328, 528]}, 'in': {'2': [3, 37, 76, 444, 851], '4': [10, 20, 110, 470, 500], '7': [5, 15, 25, 195]}, 'rush': {'2': [2, 66, 194, 321, 702], '4': [9, 69, 149, 429, 569], '7': [4, 14, 404]}, 'to': {'2': [47, 86, 234, 999], '4': [14, 24, 774, 944], '7': [199, 319, 599, 709]}, 'tread': {'2': [57, 94, 333], '4': [15, 35, 155], '7': [20, 320]}, 'where': {'2': [67, 124, 393, 1001], '4': [11, 41, 101, 421, 431], '7': [15, 35, 735]}}\n"
     ]
    }
   ],
   "source": [
    "positionalIndex = {}\n",
    "\n",
    "with open ('Position Indices.txt') as infile:\n",
    "    content = infile.read()\n",
    "    lines = content.split(';\\n')\n",
    "\n",
    "    for line in lines:\n",
    "        parts = line.split(':')\n",
    "        term = parts[0].strip()\n",
    "        #The rest of each term positional index.\n",
    "        postings = parts[1:]\n",
    "        \n",
    "        #First: extract positional index elements to a list.   \n",
    "        pos_inxList=[]\n",
    "        #Append the first element aka. doc2\n",
    "        pos_inxList.append(postings[0].strip(' '))\n",
    "        \n",
    "        for element in postings: \n",
    "            element = element.strip(' ').split(' ')\n",
    "            #Because elements are not splitted correctly.\n",
    "            if len(element) == 2:\n",
    "                babyPostings = element[0].strip(';')\n",
    "                pos_inxList.append(babyPostings)\n",
    "                docNum = element[1]\n",
    "                pos_inxList.append(docNum)\n",
    "        pos_inxList.append(element[-1])\n",
    "        \n",
    "        #Second: re-arrange positional index elements to a dictionary.\n",
    "        #Keys: Document numbers | Values: positional index.\n",
    "        pos_inxDict={}\n",
    "        for i in range(0, len(pos_inxList), 2):\n",
    "            docNumber = pos_inxList[i]\n",
    "            rawPositions = pos_inxList[i + 1].strip('<>;')\n",
    "            positions = list(map(int, rawPositions.split(',')))\n",
    "            pos_inxDict[docNumber] = positions\n",
    "        \n",
    "        #Add to the master dictionary.\n",
    "        positionalIndex[term] = pos_inxDict\n",
    "\n",
    "print(f'Positional Index data:')\n",
    "print(positionalIndex)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "30290cf8-1074-4002-8255-bd1c7a100924",
   "metadata": {},
   "source": [
    "### a. Which document(s) (if any) match each of the following queries where each expression within quotes is a phrase query?  At which positions do the queries match?"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fc0d33c5-d4cc-4588-9606-66becacf1dd7",
   "metadata": {},
   "source": [
    " #### i. “fools rush in”"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "341e073e-e37f-47c5-91eb-52152b26b580",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Matching found in document 2 at position index 1.\n",
      "Matching found in document 4 at position index 8.\n",
      "Matching found in document 7 at position index 3.\n",
      "Matching found in document 7 at position index 13.\n"
     ]
    }
   ],
   "source": [
    "query1 = 'fools rush in'\n",
    "queryTerms1 = query1.split()\n",
    "\n",
    "matchingDocs = []\n",
    "\n",
    "#Get neccessary items only (based on query terms) from positionalIndex dictionary.\n",
    "for docNum, positions in positionalIndex[queryTerms1[0]].items():\n",
    "    \n",
    "    #Get positional index of each term in the query.\n",
    "    pos1stTerm1 = positionalIndex[queryTerms1[0]].get(docNum, [])\n",
    "    pos2ndTerm1 = positionalIndex[queryTerms1[1]].get(docNum, [])\n",
    "    pos3rdTerm1 = positionalIndex[queryTerms1[2]].get(docNum, [])\n",
    "\n",
    "    #Basic logic positional index of 1st term + 1 = positional index of 2nd term.\n",
    "    for posFirst in pos1stTerm1:\n",
    "        if posFirst + 1 in pos2ndTerm1 and posFirst + 2 in pos3rdTerm1:\n",
    "            matchingDocs.append((docNum, posFirst))\n",
    "            \n",
    "for pairs in matchingDocs:\n",
    "    print(f'Matching found in document {pairs[0]} at position index {pairs[1]}.')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "67f719c6-5b43-482d-a455-fea7dee24389",
   "metadata": {},
   "source": [
    "ii. “fools rush in” AND “angels fear to tread”."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "644e3930-bc7c-4f11-810a-a2450f9f9151",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Matching found in document 4 at position indices 8 and 12.\n"
     ]
    }
   ],
   "source": [
    "#Same logic as previous part.\n",
    "query2 = 'angels fear to tread'\n",
    "queryTerms2 = query2.split()\n",
    "\n",
    "matchingDocs2 = []\n",
    "\n",
    "for docNum, positions in positionalIndex[queryTerms2[0]].items():\n",
    "  \n",
    "    pos1stTerm2 = positionalIndex[queryTerms2[0]].get(docNum, [])\n",
    "    pos2ndTerm2 = positionalIndex[queryTerms2[1]].get(docNum, [])\n",
    "    pos3rdTerm2 = positionalIndex[queryTerms2[2]].get(docNum, [])\n",
    "    pos4thTerm2 = positionalIndex[queryTerms2[3]].get(docNum, [])\n",
    "\n",
    "    for posFirst in pos1stTerm2:\n",
    "        if posFirst + 1 in pos2ndTerm2 and posFirst + 2 in pos3rdTerm2 and posFirst + 3 in pos4thTerm2:\n",
    "            matchingDocs2.append((docNum, posFirst))\n",
    "\n",
    "for pairs in matchingDocs:\n",
    "    for pairs2 in matchingDocs2:\n",
    "        #Same document.\n",
    "        if pairs[0] == pairs2[0]:\n",
    "            print(f'Matching found in document {pairs[0]} at position indices {pairs[1]} and {pairs2[1]}.')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "356b2649-6976-4661-b2b4-ad245358e5f4",
   "metadata": {},
   "source": [
    "### b. There is something wrong with this positional index. What is the problem?\n",
    "#### One position has two words registered --> IMPOSSIBLE."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "1ec748ab-8dcb-4023-afb3-614b499be8ac",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Document 7: Duplicate position 15 for terms 'where' and 'in'.\n"
     ]
    }
   ],
   "source": [
    "def checkPosition(positionalIndex):\n",
    "    '''Check if there are multiple words register to one position within a documentt.'''\n",
    "    \n",
    "    #Re-arrange the original positional index dictionary.\n",
    "    #Still as a nested dictionary.\n",
    "    posDict = {}\n",
    "\n",
    "    for term, posIndex in positionalIndex.items():\n",
    "        for document, index in posIndex.items():\n",
    "            for idx in index:\n",
    "                #New index position, add in.\n",
    "                if idx not in posDict:\n",
    "                    #Positions as keys | terms & doc number as values (dictionary type).\n",
    "                    posDict[idx] = {term: document}\n",
    "                \n",
    "                #Index position already exist in the dictionary.\n",
    "                else:\n",
    "                    #Check if current document exists as well.\n",
    "                    if document in posDict[idx].values():\n",
    "                        \n",
    "                        #Retrieve the current item with same index & document.\n",
    "                        for items in posDict[idx].items():\n",
    "                            if items[1] == document:\n",
    "                                currentTerm = items[0]\n",
    "                        print(f\"Document {document}: Duplicate position {idx} for terms '{term}' and '{currentTerm}'.\")\n",
    "                    \n",
    "                    #If different document, add to the dictionary.\n",
    "                    else:\n",
    "                        posDict[idx][term] = document\n",
    "checkPosition(positionalIndex)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bec32340-568f-4215-a2a5-57b6034300b9",
   "metadata": {},
   "source": [
    "## 2. TFiDF Weighting"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5c6a1806-d721-4034-b1f2-ec189893959a",
   "metadata": {},
   "source": [
    "### a. Compute the new weights for all the terms in documents DOC3 and DOC7 using the tf x idf approach.  Use the raw term frequency for TF (for both documents and query) and the log of base 10 of N/df for IDF.  "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "31e30161-ab21-4d99-8d9d-d624432bbf18",
   "metadata": {},
   "source": [
    "#### Create document-term matrix. Documents as rows, terms as columns. "
   ]
  },
  {
   "cell_type": "raw",
   "id": "11d2f615-4dbd-4236-b19b-299746747339",
   "metadata": {},
   "source": [
    "        Term1 Term2 Term3 Term4 Term5 Term6 Term7 Term8\n",
    "        -----------------------------------------------\n",
    "   DOC1   0     3     1     0     0     2     1     0\n",
    "   DOC2   5     0     0     0     3     0     0     2\n",
    "   DOC3   3     0     4     3     4     0     0     5\n",
    "   DOC4   1     8     0     3     0     1     4     0\n",
    "   DOC5   0     1     0     0     0     5     4     2\n",
    "   DOC6   2     0     2     0     0     4     0     1\n",
    "   DOC7   2     5     0     3     0     1     4     2\n",
    "   DOC8   3     3     0     2     0     0     1     3\n",
    "   DOC9   0     0     3     3     3     0     0     0\n",
    "   DOC10  1     0     5     0     2     4     0     2\n",
    "         ----------------------------------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "e6d041d1-45fc-4091-9ad3-87d290029cca",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Document-term Matrix shape: (10, 8)\n",
      "Number of terms: 8\n",
      "Number of documents: 10\n"
     ]
    }
   ],
   "source": [
    "#Read: 3 times Term2 occurs in DOC1.\n",
    "doc_termMatrix = np.array([[0, 3, 1, 0, 0, 2, 1, 0], \n",
    "                           [5, 0, 0, 0, 3, 0, 0, 2], \n",
    "                           [3, 0, 4, 3, 4, 0, 0, 5], \n",
    "                           [1, 8, 0, 3, 0, 1, 4, 0],\n",
    "                           [0, 1, 0, 0, 0, 5, 4, 2],\n",
    "                           [2, 0, 2, 0, 0, 4, 0, 1], \n",
    "                           [2, 5, 0, 3, 0, 1, 4, 2],\n",
    "                           [3, 3, 0, 2, 0, 0, 1, 3],\n",
    "                           [0, 0, 3, 3, 3, 0, 0, 0],\n",
    "                           [1, 0, 5, 0, 2, 4, 0, 2]])\n",
    "print(f'Document-term Matrix shape: {doc_termMatrix.shape}')\n",
    "print(f'Number of terms: {doc_termMatrix.shape[1]}')\n",
    "print(f'Number of documents: {doc_termMatrix.shape[0]}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "583fc143-896a-4bbd-bc34-4d5069e8672d",
   "metadata": {},
   "source": [
    "#### Get document frequency"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "fc1e0493-1da4-4ff0-9261-fd98b7392d41",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of documents a term exists in:\n",
      "[[7]\n",
      " [5]\n",
      " [5]\n",
      " [5]\n",
      " [4]\n",
      " [6]\n",
      " [5]\n",
      " [7]]\n"
     ]
    }
   ],
   "source": [
    "#Transpose the data to term-document matrix.\n",
    "#Terms are row, documents are columns. \n",
    "term_docMatrix = doc_termMatrix.T\n",
    "\n",
    "#Get document frequency/count of each term. \n",
    "#Counting non-zero elements for each row - axis = 1.\n",
    "#Reshaped into one column vector.\n",
    "#Read: Term 1 occurs in 7 out of 10 documents.\n",
    "docNum_perTerm = np.count_nonzero(term_docMatrix, axis=1).reshape(-1, 1)\n",
    "print('Number of documents a term exists in:')\n",
    "print(docNum_perTerm)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "20ad527d-65f0-4fab-b665-915d7f2db4f8",
   "metadata": {},
   "source": [
    "#### Create base matrix with each element is total number of documents N."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "8861e857-8cce-4ca7-9106-b8c17c154bd2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[10. 10. 10. 10. 10. 10. 10. 10. 10. 10.]\n",
      " [10. 10. 10. 10. 10. 10. 10. 10. 10. 10.]\n",
      " [10. 10. 10. 10. 10. 10. 10. 10. 10. 10.]\n",
      " [10. 10. 10. 10. 10. 10. 10. 10. 10. 10.]\n",
      " [10. 10. 10. 10. 10. 10. 10. 10. 10. 10.]\n",
      " [10. 10. 10. 10. 10. 10. 10. 10. 10. 10.]\n",
      " [10. 10. 10. 10. 10. 10. 10. 10. 10. 10.]\n",
      " [10. 10. 10. 10. 10. 10. 10. 10. 10. 10.]]\n"
     ]
    }
   ],
   "source": [
    "#Get total document counts.\n",
    "documentNum = term_docMatrix.shape[1]\n",
    "documentNum_Matrix=np.ones(np.shape(term_docMatrix), dtype=float)*documentNum\n",
    "print(documentNum_Matrix)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3c25b56d-b97f-4c31-a1f4-1cc55f62e736",
   "metadata": {},
   "source": [
    "#### Calculate IDF following the instruction. Using log base of 10."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "6120de46-1545-4282-bbdd-dc9a54ac9dbd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IDF:\n",
      "[[0.1549  0.1549  0.1549  0.1549  0.1549  0.1549  0.1549  0.1549  0.1549  0.1549 ]\n",
      " [0.30103 0.30103 0.30103 0.30103 0.30103 0.30103 0.30103 0.30103 0.30103 0.30103]\n",
      " [0.30103 0.30103 0.30103 0.30103 0.30103 0.30103 0.30103 0.30103 0.30103 0.30103]\n",
      " [0.30103 0.30103 0.30103 0.30103 0.30103 0.30103 0.30103 0.30103 0.30103 0.30103]\n",
      " [0.39794 0.39794 0.39794 0.39794 0.39794 0.39794 0.39794 0.39794 0.39794 0.39794]\n",
      " [0.22185 0.22185 0.22185 0.22185 0.22185 0.22185 0.22185 0.22185 0.22185 0.22185]\n",
      " [0.30103 0.30103 0.30103 0.30103 0.30103 0.30103 0.30103 0.30103 0.30103 0.30103]\n",
      " [0.1549  0.1549  0.1549  0.1549  0.1549  0.1549  0.1549  0.1549  0.1549  0.1549 ]]\n"
     ]
    }
   ],
   "source": [
    "#Set print out option, 5 decimals.\n",
    "np.set_printoptions(precision=5,suppress=True,linewidth=120)\n",
    "\n",
    "#ID formula log10(N/df)\n",
    "#documentNum_Matrix: total number of document. \n",
    "#docNum_perTerm: number of document a term exists in.\n",
    "IDF = np.log10(np.divide(documentNum_Matrix, docNum_perTerm))\n",
    "print('IDF:')\n",
    "print(IDF)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ff8e61fb-f8de-4ebe-ac9a-0d4094a9d4de",
   "metadata": {},
   "source": [
    "#### Calculate TFxIDF with TF as raw term frequency."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "0ff27a79-34a8-4f96-80a7-fc1ea524c04b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0.      0.77451 0.46471 0.1549  0.      0.3098  0.3098  0.46471 0.      0.1549 ]\n",
      " [0.90309 0.      0.      2.40824 0.30103 0.      1.50515 0.90309 0.      0.     ]\n",
      " [0.30103 0.      1.20412 0.      0.      0.60206 0.      0.      0.90309 1.50515]\n",
      " [0.      0.      0.90309 0.90309 0.      0.      0.90309 0.60206 0.90309 0.     ]\n",
      " [0.      1.19382 1.59176 0.      0.      0.      0.      0.      1.19382 0.79588]\n",
      " [0.4437  0.      0.      0.22185 1.10924 0.88739 0.22185 0.      0.      0.88739]\n",
      " [0.30103 0.      0.      1.20412 1.20412 0.      1.20412 0.30103 0.      0.     ]\n",
      " [0.      0.3098  0.77451 0.      0.3098  0.1549  0.3098  0.46471 0.      0.3098 ]]\n"
     ]
    }
   ],
   "source": [
    "#Compute the TFxIDF values for document-term entry.\n",
    "TFIDF = term_docMatrix * IDF\n",
    "print(TFIDF)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a0abd5c8-c646-404d-902c-01dd1f479d01",
   "metadata": {},
   "source": [
    "#### Convert back to document-term matrix."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "9e961f0f-2471-428f-9913-004d48e52d12",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.     , 0.90309, 0.30103, 0.     , 0.     , 0.4437 , 0.30103, 0.     ],\n",
       "       [0.77451, 0.     , 0.     , 0.     , 1.19382, 0.     , 0.     , 0.3098 ],\n",
       "       [0.46471, 0.     , 1.20412, 0.90309, 1.59176, 0.     , 0.     , 0.77451],\n",
       "       [0.1549 , 2.40824, 0.     , 0.90309, 0.     , 0.22185, 1.20412, 0.     ],\n",
       "       [0.     , 0.30103, 0.     , 0.     , 0.     , 1.10924, 1.20412, 0.3098 ],\n",
       "       [0.3098 , 0.     , 0.60206, 0.     , 0.     , 0.88739, 0.     , 0.1549 ],\n",
       "       [0.3098 , 1.50515, 0.     , 0.90309, 0.     , 0.22185, 1.20412, 0.3098 ],\n",
       "       [0.46471, 0.90309, 0.     , 0.60206, 0.     , 0.     , 0.30103, 0.46471],\n",
       "       [0.     , 0.     , 0.90309, 0.90309, 1.19382, 0.     , 0.     , 0.     ],\n",
       "       [0.1549 , 0.     , 1.50515, 0.     , 0.79588, 0.88739, 0.     , 0.3098 ]])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "doc_termTFIDF = TFIDF.transpose()\n",
    "doc_termTFIDF"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1eca08b7-d84c-46b0-9b83-3bf427956946",
   "metadata": {},
   "source": [
    "#### Output as pandas dataframe. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "3f87dd5f-14ee-4bd7-bfb2-ebfeb58cba55",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TF-IDF weights:\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Term1</th>\n",
       "      <th>Term2</th>\n",
       "      <th>Term3</th>\n",
       "      <th>Term4</th>\n",
       "      <th>Term5</th>\n",
       "      <th>Term6</th>\n",
       "      <th>Term7</th>\n",
       "      <th>Term8</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Doc1</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.90309</td>\n",
       "      <td>0.30103</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.443697</td>\n",
       "      <td>0.30103</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Doc2</th>\n",
       "      <td>0.774510</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>1.19382</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.309804</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Doc3</th>\n",
       "      <td>0.464706</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>1.20412</td>\n",
       "      <td>0.90309</td>\n",
       "      <td>1.59176</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.774510</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Doc4</th>\n",
       "      <td>0.154902</td>\n",
       "      <td>2.40824</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.90309</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.221849</td>\n",
       "      <td>1.20412</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Doc5</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.30103</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>1.109244</td>\n",
       "      <td>1.20412</td>\n",
       "      <td>0.309804</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Doc6</th>\n",
       "      <td>0.309804</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.60206</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.887395</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.154902</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Doc7</th>\n",
       "      <td>0.309804</td>\n",
       "      <td>1.50515</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.90309</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.221849</td>\n",
       "      <td>1.20412</td>\n",
       "      <td>0.309804</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Doc8</th>\n",
       "      <td>0.464706</td>\n",
       "      <td>0.90309</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.60206</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.30103</td>\n",
       "      <td>0.464706</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Doc9</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.90309</td>\n",
       "      <td>0.90309</td>\n",
       "      <td>1.19382</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Doc10</th>\n",
       "      <td>0.154902</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>1.50515</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.79588</td>\n",
       "      <td>0.887395</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.309804</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          Term1    Term2    Term3    Term4    Term5     Term6    Term7  \\\n",
       "Doc1   0.000000  0.90309  0.30103  0.00000  0.00000  0.443697  0.30103   \n",
       "Doc2   0.774510  0.00000  0.00000  0.00000  1.19382  0.000000  0.00000   \n",
       "Doc3   0.464706  0.00000  1.20412  0.90309  1.59176  0.000000  0.00000   \n",
       "Doc4   0.154902  2.40824  0.00000  0.90309  0.00000  0.221849  1.20412   \n",
       "Doc5   0.000000  0.30103  0.00000  0.00000  0.00000  1.109244  1.20412   \n",
       "Doc6   0.309804  0.00000  0.60206  0.00000  0.00000  0.887395  0.00000   \n",
       "Doc7   0.309804  1.50515  0.00000  0.90309  0.00000  0.221849  1.20412   \n",
       "Doc8   0.464706  0.90309  0.00000  0.60206  0.00000  0.000000  0.30103   \n",
       "Doc9   0.000000  0.00000  0.90309  0.90309  1.19382  0.000000  0.00000   \n",
       "Doc10  0.154902  0.00000  1.50515  0.00000  0.79588  0.887395  0.00000   \n",
       "\n",
       "          Term8  \n",
       "Doc1   0.000000  \n",
       "Doc2   0.309804  \n",
       "Doc3   0.774510  \n",
       "Doc4   0.000000  \n",
       "Doc5   0.309804  \n",
       "Doc6   0.154902  \n",
       "Doc7   0.309804  \n",
       "Doc8   0.464706  \n",
       "Doc9   0.000000  \n",
       "Doc10  0.309804  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "columnNames = ['Term1', 'Term2', 'Term3', 'Term4', 'Term5', 'Term6', 'Term7', 'Term8']\n",
    "indexNames = [f'Doc{i+1}' for i in range(len(doc_termTFIDF))]\n",
    "doc_termTFIDF_pd = pd.DataFrame(doc_termTFIDF, columns=columnNames, index=indexNames)\n",
    "print('TF-IDF weights:')\n",
    "doc_termTFIDF_pd"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "29c28b54-b4a4-4749-810c-94a23813156c",
   "metadata": {},
   "source": [
    "### New weights for all the terms in documents DOC3."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "5b4ca6eb-722c-4fcd-97bb-428c4b54b54b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "New weights for all the terms in documents DOC3:\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Term1    0.464706\n",
       "Term2    0.000000\n",
       "Term3    1.204120\n",
       "Term4    0.903090\n",
       "Term5    1.591760\n",
       "Term6    0.000000\n",
       "Term7    0.000000\n",
       "Term8    0.774510\n",
       "Name: Doc3, dtype: float64"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "doc3 = doc_termTFIDF_pd.loc['Doc3']\n",
    "print('New weights for all the terms in documents DOC3:')\n",
    "doc3"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "42e600e7-afce-4e3c-9e44-39fb87bb8eda",
   "metadata": {},
   "source": [
    "### New weights for all the terms in documents DOC7."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "182d8412-1e96-42eb-a175-0ad2922b6fa7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "New weights for all the terms in documents DOC7:\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Term1    0.309804\n",
       "Term2    1.505150\n",
       "Term3    0.000000\n",
       "Term4    0.903090\n",
       "Term5    0.000000\n",
       "Term6    0.221849\n",
       "Term7    1.204120\n",
       "Term8    0.309804\n",
       "Name: Doc7, dtype: float64"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "doc7 = doc_termTFIDF_pd.loc['Doc7']\n",
    "print('New weights for all the terms in documents DOC7:')\n",
    "doc7"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7e04a9cf-0435-4877-b8ac-0c4200bc3fe9",
   "metadata": {},
   "source": [
    "### b. Using cosine as the measure, which document is the most relevant to DOC7?  Show your calculations.  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "ad865264-6938-4bf9-b300-208f37409618",
   "metadata": {},
   "outputs": [],
   "source": [
    "def cosineSimilarity (instance1, instance2):\n",
    "    '''Return Cosine similarity distance between two instance.'''\n",
    "    \n",
    "    #Find the vector norm for each instance. \n",
    "    instance1_norm = np.linalg.norm(instance1)\n",
    "    instance2_norm = np.linalg.norm(instance2)\n",
    "          \n",
    "    #Compute Cosine: divide the dot product of the predicted instance versus each document instance by the product of the two norms.\n",
    "    cosine = np.dot(instance1, instance2)/(instance1_norm * instance2_norm)\n",
    "    \n",
    "    return cosine    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "09b3a9e7-99ba-4ecd-b078-37fce6c321dc",
   "metadata": {},
   "outputs": [],
   "source": [
    "def cosine_simDoc(doc, weightMatrix):\n",
    "    '''Get the most similar document based on Cosine similarity.'''\n",
    "    '''Using raw term frequnecy and pandas data frame.'''\n",
    "    \n",
    "    cosSim_res = []\n",
    "    \n",
    "    for index, row in weightMatrix.iterrows():\n",
    "        cosineSim = cosineSimilarity(doc, row.values)\n",
    "        cosSim_res.append((index, cosineSim))\n",
    "    \n",
    "    #Sort the similarities in descending order.\n",
    "    cosSim_res.sort(key=lambda sim: sim[1], reverse=True)\n",
    "    \n",
    "    #Return 2nd similar item. First item is the document itself.\n",
    "    return cosSim_res[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "064d5cde-2708-489f-85fd-c6680f36cd42",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The most relevant document to Doc7 is Doc4 with a Cosine similarity of 0.9280\n"
     ]
    }
   ],
   "source": [
    "doc_termMatrix_pd = pd.DataFrame(doc_termMatrix)\n",
    "doc7Raw = doc_termMatrix_pd.iloc[6]\n",
    "cosineSimilar = cosine_simDoc(doc7Raw, doc_termMatrix_pd)\n",
    "print(f\"The most relevant document to Doc7 is Doc{cosineSimilar[0] + 1} with a Cosine similarity of {cosineSimilar[1]:.4f}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dc5dfda0-5729-4a57-ac60-609211907f09",
   "metadata": {},
   "source": [
    "### 3. Inverted Index"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6ac6b6d6-3036-48d1-a4cf-0ed8d7210656",
   "metadata": {},
   "source": [
    "### Import dataset as pandas dataframe."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "5dcb14c6-73d0-4f86-942d-3c5eefdff0cf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Numnber of rows and columns of the ted data: (2550, 17)\n"
     ]
    }
   ],
   "source": [
    "ted = pd.read_csv('ted_main.csv', encoding='utf-8')\n",
    "print(f'Numnber of rows and columns of the ted data: {ted.shape}')\n",
    "#Read data as Pandas dataframe and get number of rows and columns. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "c74a2ce4-c38c-412f-bace-364972fe95f4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>comments</th>\n",
       "      <th>description</th>\n",
       "      <th>duration</th>\n",
       "      <th>event</th>\n",
       "      <th>film_date</th>\n",
       "      <th>languages</th>\n",
       "      <th>main_speaker</th>\n",
       "      <th>name</th>\n",
       "      <th>num_speaker</th>\n",
       "      <th>published_date</th>\n",
       "      <th>ratings</th>\n",
       "      <th>related_talks</th>\n",
       "      <th>speaker_occupation</th>\n",
       "      <th>tags</th>\n",
       "      <th>title</th>\n",
       "      <th>url</th>\n",
       "      <th>views</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4553</td>\n",
       "      <td>Sir Ken Robinson makes an entertaining and pro...</td>\n",
       "      <td>1164</td>\n",
       "      <td>TED2006</td>\n",
       "      <td>1140825600</td>\n",
       "      <td>60</td>\n",
       "      <td>Ken Robinson</td>\n",
       "      <td>Ken Robinson: Do schools kill creativity?</td>\n",
       "      <td>1</td>\n",
       "      <td>1151367060</td>\n",
       "      <td>[{'id': 7, 'name': 'Funny', 'count': 19645}, {...</td>\n",
       "      <td>[{'id': 865, 'hero': 'https://pe.tedcdn.com/im...</td>\n",
       "      <td>Author/educator</td>\n",
       "      <td>['children', 'creativity', 'culture', 'dance',...</td>\n",
       "      <td>Do schools kill creativity?</td>\n",
       "      <td>https://www.ted.com/talks/ken_robinson_says_sc...</td>\n",
       "      <td>47227110</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>265</td>\n",
       "      <td>With the same humor and humanity he exuded in ...</td>\n",
       "      <td>977</td>\n",
       "      <td>TED2006</td>\n",
       "      <td>1140825600</td>\n",
       "      <td>43</td>\n",
       "      <td>Al Gore</td>\n",
       "      <td>Al Gore: Averting the climate crisis</td>\n",
       "      <td>1</td>\n",
       "      <td>1151367060</td>\n",
       "      <td>[{'id': 7, 'name': 'Funny', 'count': 544}, {'i...</td>\n",
       "      <td>[{'id': 243, 'hero': 'https://pe.tedcdn.com/im...</td>\n",
       "      <td>Climate advocate</td>\n",
       "      <td>['alternative energy', 'cars', 'climate change...</td>\n",
       "      <td>Averting the climate crisis</td>\n",
       "      <td>https://www.ted.com/talks/al_gore_on_averting_...</td>\n",
       "      <td>3200520</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>124</td>\n",
       "      <td>New York Times columnist David Pogue takes aim...</td>\n",
       "      <td>1286</td>\n",
       "      <td>TED2006</td>\n",
       "      <td>1140739200</td>\n",
       "      <td>26</td>\n",
       "      <td>David Pogue</td>\n",
       "      <td>David Pogue: Simplicity sells</td>\n",
       "      <td>1</td>\n",
       "      <td>1151367060</td>\n",
       "      <td>[{'id': 7, 'name': 'Funny', 'count': 964}, {'i...</td>\n",
       "      <td>[{'id': 1725, 'hero': 'https://pe.tedcdn.com/i...</td>\n",
       "      <td>Technology columnist</td>\n",
       "      <td>['computers', 'entertainment', 'interface desi...</td>\n",
       "      <td>Simplicity sells</td>\n",
       "      <td>https://www.ted.com/talks/david_pogue_says_sim...</td>\n",
       "      <td>1636292</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   comments                                        description  duration  \\\n",
       "0      4553  Sir Ken Robinson makes an entertaining and pro...      1164   \n",
       "1       265  With the same humor and humanity he exuded in ...       977   \n",
       "2       124  New York Times columnist David Pogue takes aim...      1286   \n",
       "\n",
       "     event   film_date  languages  main_speaker  \\\n",
       "0  TED2006  1140825600         60  Ken Robinson   \n",
       "1  TED2006  1140825600         43       Al Gore   \n",
       "2  TED2006  1140739200         26   David Pogue   \n",
       "\n",
       "                                        name  num_speaker  published_date  \\\n",
       "0  Ken Robinson: Do schools kill creativity?            1      1151367060   \n",
       "1       Al Gore: Averting the climate crisis            1      1151367060   \n",
       "2              David Pogue: Simplicity sells            1      1151367060   \n",
       "\n",
       "                                             ratings  \\\n",
       "0  [{'id': 7, 'name': 'Funny', 'count': 19645}, {...   \n",
       "1  [{'id': 7, 'name': 'Funny', 'count': 544}, {'i...   \n",
       "2  [{'id': 7, 'name': 'Funny', 'count': 964}, {'i...   \n",
       "\n",
       "                                       related_talks    speaker_occupation  \\\n",
       "0  [{'id': 865, 'hero': 'https://pe.tedcdn.com/im...       Author/educator   \n",
       "1  [{'id': 243, 'hero': 'https://pe.tedcdn.com/im...      Climate advocate   \n",
       "2  [{'id': 1725, 'hero': 'https://pe.tedcdn.com/i...  Technology columnist   \n",
       "\n",
       "                                                tags  \\\n",
       "0  ['children', 'creativity', 'culture', 'dance',...   \n",
       "1  ['alternative energy', 'cars', 'climate change...   \n",
       "2  ['computers', 'entertainment', 'interface desi...   \n",
       "\n",
       "                         title  \\\n",
       "0  Do schools kill creativity?   \n",
       "1  Averting the climate crisis   \n",
       "2             Simplicity sells   \n",
       "\n",
       "                                                 url     views  \n",
       "0  https://www.ted.com/talks/ken_robinson_says_sc...  47227110  \n",
       "1  https://www.ted.com/talks/al_gore_on_averting_...   3200520  \n",
       "2  https://www.ted.com/talks/david_pogue_says_sim...   1636292  "
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ted.head(3)\n",
    "#Get the first 3 rows. "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "43df0a33-b08a-4a25-b40a-49747da3b53d",
   "metadata": {},
   "source": [
    "#### Extract 'description' 2nd column."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "f5e89944-d80f-41e9-8abd-102fd29b2323",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of row of the description column: (2550, 1)\n"
     ]
    }
   ],
   "source": [
    "tedDesc = ted[['description']]\n",
    "print(f'Number of row of the description column: {tedDesc.shape}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "ea599c62-842e-49ae-887b-64a4a7832909",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "description    With the same humor and humanity he exuded in ...\n",
       "Name: 1, dtype: object"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tedDesc.iloc[1]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b576d066-4a1e-4d12-a5c7-40edfeb92bfb",
   "metadata": {},
   "source": [
    "#### Extract 'url' 16th column."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "8600c60e-b939-46c5-b25b-679cbe536c96",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of row of the url column: (2550, 1)\n"
     ]
    }
   ],
   "source": [
    "tedURL = ted[['url']]\n",
    "print(f'Number of row of the url column: {tedURL.shape}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "a8f919e7-f4d9-430e-8927-876c316668be",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "url    https://www.ted.com/talks/al_gore_on_averting_...\n",
       "Name: 1, dtype: object"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tedURL.iloc[1]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ca11f87b-f064-481e-a80c-6d9e420bd473",
   "metadata": {},
   "source": [
    "#### Pre-process following scheme [C]: Word tokenization + Case folding (lower-case) + Stopword filtering + Non-alphabet filtering + Porter stemming."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "37dfcd70-2f00-4e70-b027-94e16623dfde",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Set to store English stop word for efficiency.\n",
    "stopwordsSet = set(stopwords.words('english'))\n",
    "\n",
    "#Porter stemmer object.\n",
    "porterStemmer = nltk.PorterStemmer()\n",
    "\n",
    "def processDescription(description):\n",
    "    '''Apply Word tokenization + Case folding (lower-case) + Stopword filtering + Non-alphabet filtering + Porter stemming to a string.'''\n",
    "    \n",
    "    tokens = [porterStemmer.stem(token) for token in word_tokenize(description.lower())\n",
    "              if token.isalpha() and token not in stopwordsSet]\n",
    "    \n",
    "    #Using Counter for dictionary output.\n",
    "    return Counter(tokens)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "486b17c1-4594-4d51-97f1-eaf5ffd277e2",
   "metadata": {},
   "source": [
    "### Data frame needed for this problem."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "66c630f8-093a-4606-a25e-dc00e740f714",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>url</th>\n",
       "      <th>description</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>https://www.ted.com/talks/ken_robinson_says_sc...</td>\n",
       "      <td>Sir Ken Robinson makes an entertaining and pro...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>https://www.ted.com/talks/al_gore_on_averting_...</td>\n",
       "      <td>With the same humor and humanity he exuded in ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>https://www.ted.com/talks/david_pogue_says_sim...</td>\n",
       "      <td>New York Times columnist David Pogue takes aim...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 url  \\\n",
       "0  https://www.ted.com/talks/ken_robinson_says_sc...   \n",
       "1  https://www.ted.com/talks/al_gore_on_averting_...   \n",
       "2  https://www.ted.com/talks/david_pogue_says_sim...   \n",
       "\n",
       "                                         description  \n",
       "0  Sir Ken Robinson makes an entertaining and pro...  \n",
       "1  With the same humor and humanity he exuded in ...  \n",
       "2  New York Times columnist David Pogue takes aim...  "
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tedShort = ted[['url','description']]\n",
    "tedShort.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "55938dc2-7e9d-487e-a743-9b5b5e84b675",
   "metadata": {},
   "outputs": [],
   "source": [
    "warnings.filterwarnings(\"ignore\")\n",
    "#Apply the text processing function to each description row.\n",
    "tedShort.loc[:, 'processedDesc'] = tedShort['description'].apply(processDescription)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "4a68f76d-69fa-4032-b70c-71b5df3979b2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>url</th>\n",
       "      <th>description</th>\n",
       "      <th>processedDesc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>https://www.ted.com/talks/ken_robinson_says_sc...</td>\n",
       "      <td>Sir Ken Robinson makes an entertaining and pro...</td>\n",
       "      <td>{'sir': 1, 'ken': 1, 'robinson': 1, 'make': 1,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>https://www.ted.com/talks/al_gore_on_averting_...</td>\n",
       "      <td>With the same humor and humanity he exuded in ...</td>\n",
       "      <td>{'humor': 1, 'human': 1, 'exud': 1, 'inconveni...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 url  \\\n",
       "0  https://www.ted.com/talks/ken_robinson_says_sc...   \n",
       "1  https://www.ted.com/talks/al_gore_on_averting_...   \n",
       "\n",
       "                                         description  \\\n",
       "0  Sir Ken Robinson makes an entertaining and pro...   \n",
       "1  With the same humor and humanity he exuded in ...   \n",
       "\n",
       "                                       processedDesc  \n",
       "0  {'sir': 1, 'ken': 1, 'robinson': 1, 'make': 1,...  \n",
       "1  {'humor': 1, 'human': 1, 'exud': 1, 'inconveni...  "
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tedShort.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "296d21ae-3a1d-406c-815c-dd323da901d0",
   "metadata": {},
   "source": [
    "#### Cross tabulation, each word in processed Description as a row with corresponding URL."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "ae6b0efe-929c-4482-8b5b-1e91cd0eeff7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>word</th>\n",
       "      <th>word_count</th>\n",
       "      <th>url</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>sir</td>\n",
       "      <td>1</td>\n",
       "      <td>https://www.ted.com/talks/ken_robinson_says_sc...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ken</td>\n",
       "      <td>1</td>\n",
       "      <td>https://www.ted.com/talks/ken_robinson_says_sc...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>robinson</td>\n",
       "      <td>1</td>\n",
       "      <td>https://www.ted.com/talks/ken_robinson_says_sc...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       word  word_count                                                url\n",
       "0       sir           1  https://www.ted.com/talks/ken_robinson_says_sc...\n",
       "1       ken           1  https://www.ted.com/talks/ken_robinson_says_sc...\n",
       "2  robinson           1  https://www.ted.com/talks/ken_robinson_says_sc..."
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Look at processed desciption per row, then iterate through each word.\n",
    "#Assign to tuple (A SINGLE WORD, corresponding word count within URL document, corresponding URL).\n",
    "\n",
    "pdWord = pd.DataFrame([(word, word_count, tedShort.loc[index, 'url']) for index, row in tedShort.iterrows() \n",
    "                       for word, word_count in row['processedDesc'].items()], columns=['word', 'word_count', 'url'])\n",
    "pdWord.head(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0797da2d-f4a8-4529-bf97-4cf01446a206",
   "metadata": {},
   "source": [
    "### 1. A file that maps each term name to: the term ID and the document frequency (i.e., the number of files the term occurred in). Assign a series of positive integers (1-based) to the term names. Store each term name per line. Name the file \"TED_term_index.csv\". The first couple of lines in the file should look like this:"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b9c9a8ee-8b43-4f8d-91ee-2bc60ae4ba51",
   "metadata": {},
   "source": [
    "#### Group by 'word' and count how many URL associated with a word."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "a9471d19-c881-4afe-b155-d4f9f348c190",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>word</th>\n",
       "      <th>count</th>\n",
       "      <th>term_ID</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>a</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>aakash</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>aala</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     word  count  term_ID\n",
       "0       a      2        1\n",
       "1  aakash      1        2\n",
       "2    aala      1        3"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#size() - count occurrences grouped by one word.\n",
    "pdWordcounts = pdWord.groupby('word').size().reset_index(name='count')\n",
    "pdWordcounts['term_ID'] = pdWordcounts.index + 1\n",
    "pdWordcounts.head(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "479568cd-bc25-4bb0-867c-8feca729bf04",
   "metadata": {},
   "source": [
    "#### Write to a CSV file."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "8abf4cc5-8fba-4c14-a1e2-1f79afe2f8d1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Successfully written!\n"
     ]
    }
   ],
   "source": [
    "with open('TED_term_index.csv', 'w', newline='', encoding='utf-8') as outFile:\n",
    "    outFile.write(\"word, term_ID, count\\n\")  \n",
    "    for row in pdWordcounts.itertuples(index=False):\n",
    "        template = f\"{row.word}, {row.term_ID}, {row.count}\\n\"\n",
    "        outFile.write(template)\n",
    "\n",
    "print(\"Successfully written!\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dbc9545e-7a81-40c0-9e26-20daef80608d",
   "metadata": {},
   "source": [
    "### 2. A file that maps each document ID to: the document name. Assign a series of positive integers (1-based) to the documents.Store each document ID per line. Name the file \"TED_doc_index.csv\". The first couple of lines in the file should look like this:"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a190c00d-92c4-44f7-9424-431cddf6c037",
   "metadata": {},
   "source": [
    "#### Extract url column and assign doc_ID."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "30ca2398-24ed-4b50-a636-350aa407c70f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>url</th>\n",
       "      <th>doc_ID</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>https://www.ted.com/talks/ken_robinson_says_sc...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>https://www.ted.com/talks/al_gore_on_averting_...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>https://www.ted.com/talks/david_pogue_says_sim...</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 url  doc_ID\n",
       "0  https://www.ted.com/talks/ken_robinson_says_sc...       1\n",
       "1  https://www.ted.com/talks/al_gore_on_averting_...       2\n",
       "2  https://www.ted.com/talks/david_pogue_says_sim...       3"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Add document index column.\n",
    "tedURL['doc_ID'] = tedURL.index + 1\n",
    "tedURL.head(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "deb5cfaa-43a8-45c5-be9b-5704e8b719a4",
   "metadata": {},
   "source": [
    "#### Write to a CSV file."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "a03e0578-9019-4f17-8766-63144006acad",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Successfully written!\n"
     ]
    }
   ],
   "source": [
    "with open('TED_doc_index.csv', 'w', newline='', encoding='utf-8') as outFile:\n",
    "    outFile.write(\"doc_ID, url\\n\")  \n",
    "    for row in tedURL.itertuples(index=False):\n",
    "        template = f\"{row.doc_ID}, {row.url}\"\n",
    "        outFile.write(template)\n",
    "\n",
    "print(\"Successfully written!\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d4ee3a0e-48c9-49db-a3c7-2b3fcf6843c4",
   "metadata": {},
   "source": [
    "### 3. The inverted index file that maps each term ID to its postings list. Each posting should contain a document ID and the term frequency in that document. Store one term ID per line. Name the file \"TED_inverted_index.csv\". The first couple of lines in the file should look like this:"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ecb9edfb-9e19-4554-972f-2d3353c5d653",
   "metadata": {},
   "source": [
    "#### Merge term_ID, doc_ID to main word count pandas data frame. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "047ad1f0-530b-4f41-bd05-e64673c79c62",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>word</th>\n",
       "      <th>word_count</th>\n",
       "      <th>url</th>\n",
       "      <th>term_ID</th>\n",
       "      <th>doc_ID</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>25373</th>\n",
       "      <td>a</td>\n",
       "      <td>1</td>\n",
       "      <td>https://www.ted.com/talks/lucien_engelen_crowd...</td>\n",
       "      <td>1</td>\n",
       "      <td>1146</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>65127</th>\n",
       "      <td>a</td>\n",
       "      <td>1</td>\n",
       "      <td>https://www.ted.com/talks/todd_scott_an_interg...</td>\n",
       "      <td>1</td>\n",
       "      <td>2429</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45972</th>\n",
       "      <td>aakash</td>\n",
       "      <td>1</td>\n",
       "      <td>https://www.ted.com/talks/aakash_odedra_a_danc...</td>\n",
       "      <td>2</td>\n",
       "      <td>1878</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         word  word_count                                                url  \\\n",
       "25373       a           1  https://www.ted.com/talks/lucien_engelen_crowd...   \n",
       "65127       a           1  https://www.ted.com/talks/todd_scott_an_interg...   \n",
       "45972  aakash           1  https://www.ted.com/talks/aakash_odedra_a_danc...   \n",
       "\n",
       "       term_ID  doc_ID  \n",
       "25373        1    1146  \n",
       "65127        1    2429  \n",
       "45972        2    1878  "
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Create a copy for this problem.\n",
    "pdWord_inverted=pdWord\n",
    "\n",
    "#Merge each word to correpoding term_ID.\n",
    "pdWord_inverted = pdWord_inverted.merge(pdWordcounts[['word', 'term_ID']], on='word', how='left')\n",
    "\n",
    "#Merge each doc to correpoding doc_ID.\n",
    "pdWord_inverted = pdWord_inverted.merge(tedURL[['url', 'doc_ID']], on='url', how='left')\n",
    "\n",
    "#Sorted in ascending order. \n",
    "pdWord_inverted = pdWord_inverted.sort_values(by=['term_ID', 'doc_ID'])\n",
    "\n",
    "#Display the updated pdWord DataFrame\n",
    "pdWord_inverted.head(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3cf7dbf3-9606-48d8-9d69-01623ecf3b44",
   "metadata": {},
   "source": [
    "#### Write to a CSV file with print template. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "462e82ad-fc3a-42a1-aa46-92a72822edd2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Successfully written!\n"
     ]
    }
   ],
   "source": [
    "invertedIndex_lines = []\n",
    "\n",
    "#Create template line. \n",
    "for termID, postings in pdWord_inverted.groupby('term_ID'):\n",
    "    template = f\"{termID}, \" + ', '.join(f\"{docID}, {wordCount}\" for (docID, wordCount) in postings[['doc_ID', 'word_count']].itertuples(index=False))\n",
    "    invertedIndex_lines.append(template)\n",
    "    \n",
    "with open('TED_inverted_index.csv', 'w', newline='', encoding='utf-8') as outFile:\n",
    "    outFile.write('\\n'.join(invertedIndex_lines))\n",
    "\n",
    "print(\"Successfully written!\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5bea3186-fcfd-41a3-a7d6-6f46b38b0fc1",
   "metadata": {},
   "source": [
    "## Boolean conjunctive queries.\n",
    "#### Work on pdWord_inverted pandas data frame."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "63ec0dde-6bf4-4bf7-8af1-d0e4d0a43bb7",
   "metadata": {},
   "outputs": [],
   "source": [
    "def booleanAND(term1, term2, dataFrame):\n",
    "    '''Apply boolean query search using AND gate for 2 terms'''\n",
    "    '''Return number of documents satisified condition and breakdown data.'''\n",
    "    \n",
    "    #Porter stemmer object.\n",
    "    porterStemmer = nltk.PorterStemmer()\n",
    "\n",
    "    stemmedTerm1 = porterStemmer.stem(term1)\n",
    "    stemmedTerm2 = porterStemmer.stem(term2)\n",
    "  \n",
    "    #Extract only rows with query term.\n",
    "    res1 = dataFrame[dataFrame['word'].isin([stemmedTerm1, stemmedTerm2])]\n",
    "    #Filter only rows with doc_ID occurs more than twice for 1 word.\n",
    "    res2 = res1.groupby('doc_ID').filter(lambda x: len(x) >= 2)\n",
    "    \n",
    "    #Compile output data frame.\n",
    "    res3 = pd.DataFrame(columns=['doc_ID', 'url', 'term_frequency'])\n",
    "    for doc_ID, postings in res2.groupby('doc_ID'):\n",
    "        #Make sure same URL has 1 doc_ID.\n",
    "        url = postings['url'].iloc[0]  \n",
    "        tf = postings[['word', 'word_count']].values.tolist()\n",
    "        res3 = res3.append({'doc_ID': doc_ID, 'url': url, 'term_frequency': tf}, ignore_index=True)\n",
    "    \n",
    "    docNum = res3.shape[0]\n",
    "    \n",
    "    print(f\"Number of documents have both terms '{term1}' AND '{term2}' are {docNum}.\")\n",
    "    \n",
    "    #Print out res2 for code verification. Technically we do not need. \n",
    "    return res2, res3"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "316506e6-800e-47c0-99c6-8f5b1f53b6b8",
   "metadata": {},
   "source": [
    "### a. 'climate' AND 'change'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "60371f3d-b399-452c-b252-a63ba347b6b6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of documents have both terms 'climate' AND 'change' are 35.\n"
     ]
    }
   ],
   "source": [
    "climateChange2, climateChange3 = booleanAND('climate', 'change', pdWord_inverted)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "f85bc3d2-962d-46b9-b714-35126b9951b5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Break down chart for query 'climate' AND 'change'.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>doc_ID</th>\n",
       "      <th>url</th>\n",
       "      <th>term_frequency</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "      <td>https://www.ted.com/talks/al_gore_on_averting_...</td>\n",
       "      <td>[[chang, 1], [climat, 1]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>110</td>\n",
       "      <td>https://www.ted.com/talks/john_doerr_sees_salv...</td>\n",
       "      <td>[[chang, 1], [climat, 1]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>161</td>\n",
       "      <td>https://www.ted.com/talks/david_keith_s_surpri...</td>\n",
       "      <td>[[chang, 1], [climat, 1]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>215</td>\n",
       "      <td>https://www.ted.com/talks/al_gore_s_new_thinki...</td>\n",
       "      <td>[[chang, 1], [climat, 1]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>492</td>\n",
       "      <td>https://www.ted.com/talks/gordon_brown\\n</td>\n",
       "      <td>[[chang, 1], [climat, 1]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>505</td>\n",
       "      <td>https://www.ted.com/talks/cary_fowler_one_seed...</td>\n",
       "      <td>[[chang, 1], [climat, 1]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>510</td>\n",
       "      <td>https://www.ted.com/talks/james_balog_time_lap...</td>\n",
       "      <td>[[chang, 1], [climat, 1]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>551</td>\n",
       "      <td>https://www.ted.com/talks/rachel_pike_the_scie...</td>\n",
       "      <td>[[chang, 1], [climat, 1]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>742</td>\n",
       "      <td>https://www.ted.com/talks/lewis_pugh_s_mind_sh...</td>\n",
       "      <td>[[chang, 1], [climat, 1]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>751</td>\n",
       "      <td>https://www.ted.com/talks/lee_hotz_inside_an_a...</td>\n",
       "      <td>[[chang, 1], [climat, 1]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>783</td>\n",
       "      <td>https://www.ted.com/talks/tim_jackson_s_econom...</td>\n",
       "      <td>[[chang, 1], [climat, 1]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>1167</td>\n",
       "      <td>https://www.ted.com/talks/james_hansen_why_i_m...</td>\n",
       "      <td>[[chang, 2], [climat, 2]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>1200</td>\n",
       "      <td>https://www.ted.com/talks/jonathan_foley_the_o...</td>\n",
       "      <td>[[chang, 1], [climat, 1]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>1346</td>\n",
       "      <td>https://www.ted.com/talks/vicki_arroyo_let_s_p...</td>\n",
       "      <td>[[chang, 1], [climat, 1]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>1449</td>\n",
       "      <td>https://www.ted.com/talks/allan_savory_how_to_...</td>\n",
       "      <td>[[chang, 1], [climat, 1]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>1596</td>\n",
       "      <td>https://www.ted.com/talks/michael_porter_why_b...</td>\n",
       "      <td>[[chang, 1], [climat, 1]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>1736</td>\n",
       "      <td>https://www.ted.com/talks/gavin_schmidt_the_em...</td>\n",
       "      <td>[[chang, 2], [climat, 3]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>1826</td>\n",
       "      <td>https://www.ted.com/talks/lord_nicholas_stern_...</td>\n",
       "      <td>[[chang, 1], [climat, 3]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>1895</td>\n",
       "      <td>https://www.ted.com/talks/tasso_azevedo_hopefu...</td>\n",
       "      <td>[[chang, 1], [climat, 1]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>2052</td>\n",
       "      <td>https://www.ted.com/talks/mary_robinson_why_cl...</td>\n",
       "      <td>[[chang, 1], [climat, 2]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>2059</td>\n",
       "      <td>https://www.ted.com/talks/alice_bows_larkin_we...</td>\n",
       "      <td>[[chang, 1], [climat, 2]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>2064</td>\n",
       "      <td>https://www.ted.com/talks/michael_green_how_we...</td>\n",
       "      <td>[[chang, 1], [climat, 1]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>2089</td>\n",
       "      <td>https://www.ted.com/talks/anote_tong_my_countr...</td>\n",
       "      <td>[[chang, 3], [climat, 3]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>2114</td>\n",
       "      <td>https://www.ted.com/talks/david_sedlak_4_ways_...</td>\n",
       "      <td>[[chang, 1], [climat, 2]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>2120</td>\n",
       "      <td>https://www.ted.com/talks/jill_heinerth_the_my...</td>\n",
       "      <td>[[chang, 1], [climat, 1]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>2121</td>\n",
       "      <td>https://www.ted.com/talks/jill_farrant_how_we_...</td>\n",
       "      <td>[[chang, 1], [climat, 1]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>2145</td>\n",
       "      <td>https://www.ted.com/talks/al_gore_the_case_for...</td>\n",
       "      <td>[[chang, 1], [climat, 1]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>2182</td>\n",
       "      <td>https://www.ted.com/talks/hugh_evans_what_does...</td>\n",
       "      <td>[[chang, 1], [climat, 1]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>2187</td>\n",
       "      <td>https://www.ted.com/talks/christiana_figueres_...</td>\n",
       "      <td>[[chang, 1], [climat, 3]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>2196</td>\n",
       "      <td>https://www.ted.com/talks/michael_metcalfe_a_p...</td>\n",
       "      <td>[[chang, 1], [climat, 1]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>2257</td>\n",
       "      <td>https://www.ted.com/talks/molly_winter_the_tab...</td>\n",
       "      <td>[[chang, 1], [climat, 1]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>2450</td>\n",
       "      <td>https://www.ted.com/talks/kate_stafford_how_hu...</td>\n",
       "      <td>[[chang, 1], [climat, 1]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>2462</td>\n",
       "      <td>https://www.ted.com/talks/triona_mcgrath_how_p...</td>\n",
       "      <td>[[chang, 2], [climat, 1]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>2489</td>\n",
       "      <td>https://www.ted.com/talks/kate_marvel_can_clou...</td>\n",
       "      <td>[[chang, 1], [climat, 2]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>2514</td>\n",
       "      <td>https://www.ted.com/talks/peter_calthorpe_7_pr...</td>\n",
       "      <td>[[chang, 1], [climat, 1]]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   doc_ID                                                url  \\\n",
       "0       2  https://www.ted.com/talks/al_gore_on_averting_...   \n",
       "1     110  https://www.ted.com/talks/john_doerr_sees_salv...   \n",
       "2     161  https://www.ted.com/talks/david_keith_s_surpri...   \n",
       "3     215  https://www.ted.com/talks/al_gore_s_new_thinki...   \n",
       "4     492           https://www.ted.com/talks/gordon_brown\\n   \n",
       "5     505  https://www.ted.com/talks/cary_fowler_one_seed...   \n",
       "6     510  https://www.ted.com/talks/james_balog_time_lap...   \n",
       "7     551  https://www.ted.com/talks/rachel_pike_the_scie...   \n",
       "8     742  https://www.ted.com/talks/lewis_pugh_s_mind_sh...   \n",
       "9     751  https://www.ted.com/talks/lee_hotz_inside_an_a...   \n",
       "10    783  https://www.ted.com/talks/tim_jackson_s_econom...   \n",
       "11   1167  https://www.ted.com/talks/james_hansen_why_i_m...   \n",
       "12   1200  https://www.ted.com/talks/jonathan_foley_the_o...   \n",
       "13   1346  https://www.ted.com/talks/vicki_arroyo_let_s_p...   \n",
       "14   1449  https://www.ted.com/talks/allan_savory_how_to_...   \n",
       "15   1596  https://www.ted.com/talks/michael_porter_why_b...   \n",
       "16   1736  https://www.ted.com/talks/gavin_schmidt_the_em...   \n",
       "17   1826  https://www.ted.com/talks/lord_nicholas_stern_...   \n",
       "18   1895  https://www.ted.com/talks/tasso_azevedo_hopefu...   \n",
       "19   2052  https://www.ted.com/talks/mary_robinson_why_cl...   \n",
       "20   2059  https://www.ted.com/talks/alice_bows_larkin_we...   \n",
       "21   2064  https://www.ted.com/talks/michael_green_how_we...   \n",
       "22   2089  https://www.ted.com/talks/anote_tong_my_countr...   \n",
       "23   2114  https://www.ted.com/talks/david_sedlak_4_ways_...   \n",
       "24   2120  https://www.ted.com/talks/jill_heinerth_the_my...   \n",
       "25   2121  https://www.ted.com/talks/jill_farrant_how_we_...   \n",
       "26   2145  https://www.ted.com/talks/al_gore_the_case_for...   \n",
       "27   2182  https://www.ted.com/talks/hugh_evans_what_does...   \n",
       "28   2187  https://www.ted.com/talks/christiana_figueres_...   \n",
       "29   2196  https://www.ted.com/talks/michael_metcalfe_a_p...   \n",
       "30   2257  https://www.ted.com/talks/molly_winter_the_tab...   \n",
       "31   2450  https://www.ted.com/talks/kate_stafford_how_hu...   \n",
       "32   2462  https://www.ted.com/talks/triona_mcgrath_how_p...   \n",
       "33   2489  https://www.ted.com/talks/kate_marvel_can_clou...   \n",
       "34   2514  https://www.ted.com/talks/peter_calthorpe_7_pr...   \n",
       "\n",
       "               term_frequency  \n",
       "0   [[chang, 1], [climat, 1]]  \n",
       "1   [[chang, 1], [climat, 1]]  \n",
       "2   [[chang, 1], [climat, 1]]  \n",
       "3   [[chang, 1], [climat, 1]]  \n",
       "4   [[chang, 1], [climat, 1]]  \n",
       "5   [[chang, 1], [climat, 1]]  \n",
       "6   [[chang, 1], [climat, 1]]  \n",
       "7   [[chang, 1], [climat, 1]]  \n",
       "8   [[chang, 1], [climat, 1]]  \n",
       "9   [[chang, 1], [climat, 1]]  \n",
       "10  [[chang, 1], [climat, 1]]  \n",
       "11  [[chang, 2], [climat, 2]]  \n",
       "12  [[chang, 1], [climat, 1]]  \n",
       "13  [[chang, 1], [climat, 1]]  \n",
       "14  [[chang, 1], [climat, 1]]  \n",
       "15  [[chang, 1], [climat, 1]]  \n",
       "16  [[chang, 2], [climat, 3]]  \n",
       "17  [[chang, 1], [climat, 3]]  \n",
       "18  [[chang, 1], [climat, 1]]  \n",
       "19  [[chang, 1], [climat, 2]]  \n",
       "20  [[chang, 1], [climat, 2]]  \n",
       "21  [[chang, 1], [climat, 1]]  \n",
       "22  [[chang, 3], [climat, 3]]  \n",
       "23  [[chang, 1], [climat, 2]]  \n",
       "24  [[chang, 1], [climat, 1]]  \n",
       "25  [[chang, 1], [climat, 1]]  \n",
       "26  [[chang, 1], [climat, 1]]  \n",
       "27  [[chang, 1], [climat, 1]]  \n",
       "28  [[chang, 1], [climat, 3]]  \n",
       "29  [[chang, 1], [climat, 1]]  \n",
       "30  [[chang, 1], [climat, 1]]  \n",
       "31  [[chang, 1], [climat, 1]]  \n",
       "32  [[chang, 2], [climat, 1]]  \n",
       "33  [[chang, 1], [climat, 2]]  \n",
       "34  [[chang, 1], [climat, 1]]  "
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\"Break down chart for query 'climate' AND 'change'.\")\n",
    "climateChange3"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "efdca868-f789-440c-9538-d492bbea12c9",
   "metadata": {},
   "source": [
    "### b.'climate' AND 'fuel'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "8fc2c11d-7cdc-4aa9-854c-13e983b1d80c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of documents have both terms 'climate' AND 'fuel' are 1.\n"
     ]
    }
   ],
   "source": [
    "climateFuel2, climateFuel3 = booleanAND('climate', 'fuel', pdWord_inverted)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "61b0414a-67f9-4f30-b27f-c5d3e680b786",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Break down chart for query 'climate' AND 'fuel'.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>doc_ID</th>\n",
       "      <th>url</th>\n",
       "      <th>term_frequency</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2260</td>\n",
       "      <td>https://www.ted.com/talks/monica_araya_a_small...</td>\n",
       "      <td>[[climat, 1], [fuel, 1]]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  doc_ID                                                url  \\\n",
       "0   2260  https://www.ted.com/talks/monica_araya_a_small...   \n",
       "\n",
       "             term_frequency  \n",
       "0  [[climat, 1], [fuel, 1]]  "
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\"Break down chart for query 'climate' AND 'fuel'.\")\n",
    "climateFuel3"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "498e7ac0-7a7b-497c-aff6-002626ae5669",
   "metadata": {},
   "source": [
    "### c. 'artificial' AND 'intelligence'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "38f60430-d83a-419d-98f8-ee8f26062209",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of documents have both terms 'artificial' AND 'intelligence' are 5.\n"
     ]
    }
   ],
   "source": [
    "artiIntel2, artiIntel3 = booleanAND('artificial', 'intelligence', pdWord_inverted)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "b34a8464-0ca1-4caa-885c-80dfe763eb0b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Break down chart for query 'artificial' AND 'intelligence'.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>doc_ID</th>\n",
       "      <th>url</th>\n",
       "      <th>term_frequency</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1268</td>\n",
       "      <td>https://www.ted.com/talks/peter_norvig_the_100...</td>\n",
       "      <td>[[artifici, 1], [intellig, 1]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1966</td>\n",
       "      <td>https://www.ted.com/talks/nick_bostrom_what_ha...</td>\n",
       "      <td>[[artifici, 1], [intellig, 2]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2319</td>\n",
       "      <td>https://www.ted.com/talks/tim_leberecht_4_ways...</td>\n",
       "      <td>[[artifici, 1], [intellig, 1]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2387</td>\n",
       "      <td>https://www.ted.com/talks/grady_booch_don_t_fe...</td>\n",
       "      <td>[[artifici, 1], [intellig, 1]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2548</td>\n",
       "      <td>https://www.ted.com/talks/radhika_nagpal_what_...</td>\n",
       "      <td>[[artifici, 1], [intellig, 2]]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  doc_ID                                                url  \\\n",
       "0   1268  https://www.ted.com/talks/peter_norvig_the_100...   \n",
       "1   1966  https://www.ted.com/talks/nick_bostrom_what_ha...   \n",
       "2   2319  https://www.ted.com/talks/tim_leberecht_4_ways...   \n",
       "3   2387  https://www.ted.com/talks/grady_booch_don_t_fe...   \n",
       "4   2548  https://www.ted.com/talks/radhika_nagpal_what_...   \n",
       "\n",
       "                   term_frequency  \n",
       "0  [[artifici, 1], [intellig, 1]]  \n",
       "1  [[artifici, 1], [intellig, 2]]  \n",
       "2  [[artifici, 1], [intellig, 1]]  \n",
       "3  [[artifici, 1], [intellig, 1]]  \n",
       "4  [[artifici, 1], [intellig, 2]]  "
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\"Break down chart for query 'artificial' AND 'intelligence'.\")\n",
    "artiIntel3"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "95e50c68-e231-4cf0-a651-aff5f7ca4c98",
   "metadata": {},
   "source": [
    "### d. 'giant' AND 'troll'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "e1941d30-2a3e-4ae9-8f65-2fd4862c41a6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of documents have both terms 'giant' AND 'troll' are 0.\n"
     ]
    }
   ],
   "source": [
    "giantTroll2, giantTroll3 = booleanAND('giant', 'troll', pdWord_inverted)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "f1f42134-3b51-43a8-af69-6149bbb649a2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Break down chart for query 'giant' AND 'troll'.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>doc_ID</th>\n",
       "      <th>url</th>\n",
       "      <th>term_frequency</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [doc_ID, url, term_frequency]\n",
       "Index: []"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\"Break down chart for query 'giant' AND 'troll'.\")\n",
    "giantTroll3"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d3f066fb-9d2a-444d-ba5b-62b67ec01f08",
   "metadata": {},
   "source": [
    "### Code verification.\n",
    "#### Let's use 'climate' AND 'change' with most output.  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "6ddd72d8-5c50-4fff-8ceb-d6b2411ffa3d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>word</th>\n",
       "      <th>word_count</th>\n",
       "      <th>url</th>\n",
       "      <th>term_ID</th>\n",
       "      <th>doc_ID</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>chang</td>\n",
       "      <td>1</td>\n",
       "      <td>https://www.ted.com/talks/al_gore_on_averting_...</td>\n",
       "      <td>1561</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2032</th>\n",
       "      <td>chang</td>\n",
       "      <td>1</td>\n",
       "      <td>https://www.ted.com/talks/john_doerr_sees_salv...</td>\n",
       "      <td>1561</td>\n",
       "      <td>110</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2946</th>\n",
       "      <td>chang</td>\n",
       "      <td>1</td>\n",
       "      <td>https://www.ted.com/talks/david_keith_s_surpri...</td>\n",
       "      <td>1561</td>\n",
       "      <td>161</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3964</th>\n",
       "      <td>chang</td>\n",
       "      <td>1</td>\n",
       "      <td>https://www.ted.com/talks/al_gore_s_new_thinki...</td>\n",
       "      <td>1561</td>\n",
       "      <td>215</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       word  word_count                                                url  \\\n",
       "27    chang           1  https://www.ted.com/talks/al_gore_on_averting_...   \n",
       "2032  chang           1  https://www.ted.com/talks/john_doerr_sees_salv...   \n",
       "2946  chang           1  https://www.ted.com/talks/david_keith_s_surpri...   \n",
       "3964  chang           1  https://www.ted.com/talks/al_gore_s_new_thinki...   \n",
       "\n",
       "      term_ID  doc_ID  \n",
       "27       1561       2  \n",
       "2032     1561     110  \n",
       "2946     1561     161  \n",
       "3964     1561     215  "
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#climateChange2 should give all doc_ID that both query words existed in. \n",
    "climateChange2.head(4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "87734381-024c-4afc-8443-85dec2c3142d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>word</th>\n",
       "      <th>word_count</th>\n",
       "      <th>url</th>\n",
       "      <th>term_ID</th>\n",
       "      <th>doc_ID</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>chang</td>\n",
       "      <td>1</td>\n",
       "      <td>https://www.ted.com/talks/al_gore_on_averting_...</td>\n",
       "      <td>1561</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>climat</td>\n",
       "      <td>1</td>\n",
       "      <td>https://www.ted.com/talks/al_gore_on_averting_...</td>\n",
       "      <td>1739</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2032</th>\n",
       "      <td>chang</td>\n",
       "      <td>1</td>\n",
       "      <td>https://www.ted.com/talks/john_doerr_sees_salv...</td>\n",
       "      <td>1561</td>\n",
       "      <td>110</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2031</th>\n",
       "      <td>climat</td>\n",
       "      <td>1</td>\n",
       "      <td>https://www.ted.com/talks/john_doerr_sees_salv...</td>\n",
       "      <td>1739</td>\n",
       "      <td>110</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2946</th>\n",
       "      <td>chang</td>\n",
       "      <td>1</td>\n",
       "      <td>https://www.ted.com/talks/david_keith_s_surpri...</td>\n",
       "      <td>1561</td>\n",
       "      <td>161</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2945</th>\n",
       "      <td>climat</td>\n",
       "      <td>1</td>\n",
       "      <td>https://www.ted.com/talks/david_keith_s_surpri...</td>\n",
       "      <td>1739</td>\n",
       "      <td>161</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3964</th>\n",
       "      <td>chang</td>\n",
       "      <td>1</td>\n",
       "      <td>https://www.ted.com/talks/al_gore_s_new_thinki...</td>\n",
       "      <td>1561</td>\n",
       "      <td>215</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3963</th>\n",
       "      <td>climat</td>\n",
       "      <td>1</td>\n",
       "      <td>https://www.ted.com/talks/al_gore_s_new_thinki...</td>\n",
       "      <td>1739</td>\n",
       "      <td>215</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9687</th>\n",
       "      <td>climat</td>\n",
       "      <td>1</td>\n",
       "      <td>https://www.ted.com/talks/gordon_brown\\n</td>\n",
       "      <td>1739</td>\n",
       "      <td>492</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9688</th>\n",
       "      <td>chang</td>\n",
       "      <td>1</td>\n",
       "      <td>https://www.ted.com/talks/gordon_brown\\n</td>\n",
       "      <td>1561</td>\n",
       "      <td>492</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        word  word_count                                                url  \\\n",
       "27     chang           1  https://www.ted.com/talks/al_gore_on_averting_...   \n",
       "26    climat           1  https://www.ted.com/talks/al_gore_on_averting_...   \n",
       "2032   chang           1  https://www.ted.com/talks/john_doerr_sees_salv...   \n",
       "2031  climat           1  https://www.ted.com/talks/john_doerr_sees_salv...   \n",
       "2946   chang           1  https://www.ted.com/talks/david_keith_s_surpri...   \n",
       "2945  climat           1  https://www.ted.com/talks/david_keith_s_surpri...   \n",
       "3964   chang           1  https://www.ted.com/talks/al_gore_s_new_thinki...   \n",
       "3963  climat           1  https://www.ted.com/talks/al_gore_s_new_thinki...   \n",
       "9687  climat           1           https://www.ted.com/talks/gordon_brown\\n   \n",
       "9688   chang           1           https://www.ted.com/talks/gordon_brown\\n   \n",
       "\n",
       "      term_ID  doc_ID  \n",
       "27       1561       2  \n",
       "26       1739       2  \n",
       "2032     1561     110  \n",
       "2031     1739     110  \n",
       "2946     1561     161  \n",
       "2945     1739     161  \n",
       "3964     1561     215  \n",
       "3963     1739     215  \n",
       "9687     1739     492  \n",
       "9688     1561     492  "
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Sort by doc_ID, expectation would be every two rows same doc_ID. \n",
    "validate = climateChange2.sort_values(by='doc_ID')\n",
    "#At this point I can confirm my code and logic are correct. \n",
    "validate.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "13c6bc2f-d8de-44ce-beac-3c581189396b",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}