Data_Preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "4da05e37",
   "metadata": {},
   "source": [
    "# NLP - Cleaning and Preprocessing Text Data of User Reviews in AppStore"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "2f86a077",
   "metadata": {},
   "outputs": [],
   "source": [
    "# pandas\n",
    "import pandas as pd\n",
    "# natural language toolkit\n",
    "import nltk\n",
    "# string for punctuation list\n",
    "import string\n",
    "# to remove links, numbers\n",
    "import re\n",
    "# to get stopwords from smart stopword list link\n",
    "from urllib.request import urlopen\n",
    "# wordnet for part of the speech\n",
    "from nltk.corpus import wordnet\n",
    "from collections import Counter\n",
    "# Tokenizer\n",
    "from nltk.tokenize import RegexpTokenizer\n",
    "# Lemmatizer\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "#Stemmers\n",
    "from nltk.stem.porter import PorterStemmer\n",
    "from nltk.stem import LancasterStemmer\n",
    "from nltk.stem.snowball import SnowballStemmer\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "77a7341d",
   "metadata": {},
   "source": [
    "##  CSV Read and DataFrame Creation\n",
    "\n",
    "We load a CSV file, create a DataFrame, and verify its shape. Initially, we have a dataset with 3097 rows and 16 columns, where each row represents a distinct reviews posted on AppStore for 10 different apps."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "ed7ded14",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_data(file):\n",
    "    data = pd.read_csv(file)\n",
    "    print(data.shape)\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "a5d02253",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(3101, 13)\n",
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 3101 entries, 0 to 3100\n",
      "Data columns (total 13 columns):\n",
      " #   Column                Non-Null Count  Dtype \n",
      "---  ------                --------------  ----- \n",
      " 0   reviewId              3101 non-null   object\n",
      " 1   userName              3101 non-null   object\n",
      " 2   userImage             3101 non-null   object\n",
      " 3   content               3101 non-null   object\n",
      " 4   score                 3101 non-null   int64 \n",
      " 5   thumbsUpCount         3101 non-null   int64 \n",
      " 6   reviewCreatedVersion  2584 non-null   object\n",
      " 7   at                    3101 non-null   object\n",
      " 8   replyContent          428 non-null    object\n",
      " 9   repliedAt             429 non-null    object\n",
      " 10  app_name              3101 non-null   object\n",
      " 11  cat1                  3101 non-null   object\n",
      " 12  inex1                 2049 non-null   object\n",
      "dtypes: int64(2), object(11)\n",
      "memory usage: 315.1+ KB\n"
     ]
    }
   ],
   "source": [
    "file = \"gps_reannotation-full.csv\"\n",
    "df = get_data(file)\n",
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "65a62398",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Apps: tiktok, facebook, uber, zoom, vinted, alexa, googlehome, linkedin, instagram, youtube\n",
      "\n",
      "Raised Ethical Concerns:  Addiction (internal), Other, inappropriate content (internal), accountability  (can be internal or external), privacy, discrimination (can be internal or external), spreading false information (internal), censorship (internal), Cyberbullying/toxicity (internal), safety (can be internal or external), identity theft (internal), scam  (can be internal or external), harmful advertising (internal), Noise, transparency  (can be internal or external), accessibility   (can be internal or external), privacy (can be external or internal), Content theft, transparency, accountability, scam, none, sustainability, safety, accessibility, censorship\n"
     ]
    }
   ],
   "source": [
    "# Get unique values of apps and raised ethical concerns of reviews\n",
    "apps = df['app_name'].unique()\n",
    "print('Apps:', ', '.join(apps))\n",
    "\n",
    "concerns = df['cat1'].unique()\n",
    "print('\\nRaised Ethical Concerns: ', ', '.join(concerns))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3f4fd8f0",
   "metadata": {},
   "source": [
    "## Remove links"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "f02c74a1",
   "metadata": {},
   "outputs": [],
   "source": [
    "def removeLink(text):\n",
    "    no_link = ' '.join(re.sub(\"(w+://S+)\", \" \", text).split())\n",
    "    return no_link"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "dafcd6ab",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0       This is yhr best app ever im littetally addict...\n",
       "1       like tik tok because it gives people a chance ...\n",
       "2                        I love this app like im addicted\n",
       "3       This app allows pedophile acts, underage half ...\n",
       "4       A very good app but I dont like how theres soo...\n",
       "                              ...                        \n",
       "3096    Great app if you don't mind getting ripped off...\n",
       "3097    Myself and my colleagues wanted to car pool an...\n",
       "3098    Its 2020 and dark mode is not even rolled out ...\n",
       "3099    Instagram give a room to share my experiences ...\n",
       "3100    CAUTION THIS APP DOESNT ALLOW YOU TO DELETE YO...\n",
       "Name: clean_content, Length: 3101, dtype: object"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['clean_content'] = df['content'].apply(lambda x: removeLink(x))\n",
    "df['clean_content']"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1c765aea",
   "metadata": {},
   "source": [
    "## Remove numbers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "33b4b7be",
   "metadata": {},
   "outputs": [],
   "source": [
    "def removeNumber(text):\n",
    "    return ' '.join(re.sub(r'[0-9]',' ', text).split())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "1bdf0cdd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0       This is yhr best app ever im littetally addict...\n",
       "1       like tik tok because it gives people a chance ...\n",
       "2                        I love this app like im addicted\n",
       "3       This app allows pedophile acts, underage half ...\n",
       "4       A very good app but I dont like how theres soo...\n",
       "                              ...                        \n",
       "3096    Great app if you don't mind getting ripped off...\n",
       "3097    Myself and my colleagues wanted to car pool an...\n",
       "3098    Its and dark mode is not even rolled out for e...\n",
       "3099    Instagram give a room to share my experiences ...\n",
       "3100    CAUTION THIS APP DOESNT ALLOW YOU TO DELETE YO...\n",
       "Name: clean_content, Length: 3101, dtype: object"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['clean_content'] = df['clean_content'].apply(lambda x: removeNumber(x))\n",
    "\n",
    "df['clean_content']"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ba6723b9",
   "metadata": {},
   "source": [
    "## Remove Emojis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "e026f425",
   "metadata": {},
   "outputs": [],
   "source": [
    "def deEmojify(text):\n",
    "    return text.encode('ascii', 'ignore').decode('ascii')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "45dce92b",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['janb mera ubar acont galti se band ho gya h plzz usy dobara bahl karva de'\n",
      " 'janb mera ubar acont galti se band ho gya h plzz usy dobara bahl karva de']\n"
     ]
    }
   ],
   "source": [
    "df['clean_content'] = df['clean_content'].apply(lambda x: deEmojify(x))\n",
    "\n",
    "#df['clean_content']\n",
    "print(df.loc[450, ['content','clean_content']].values)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3d2023f3",
   "metadata": {},
   "source": [
    "## Converting all characters to lowercase"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "4d9bc47a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0       this is yhr best app ever im littetally addict...\n",
       "1       like tik tok because it gives people a chance ...\n",
       "2                        i love this app like im addicted\n",
       "3       this app allows pedophile acts, underage half ...\n",
       "4       a very good app but i dont like how theres soo...\n",
       "                              ...                        \n",
       "3096    great app if you don't mind getting ripped off...\n",
       "3097    myself and my colleagues wanted to car pool an...\n",
       "3098    its and dark mode is not even rolled out for e...\n",
       "3099    instagram give a room to share my experiences ...\n",
       "3100    caution this app doesnt allow you to delete yo...\n",
       "Name: clean_content, Length: 3101, dtype: object"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['clean_content'] = df['clean_content'].apply(lambda x: x.lower())\n",
    "df['clean_content']"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "094c95fb",
   "metadata": {},
   "source": [
    "## Remove stopwords\n",
    "* nltk.corpus.stopwords.words('english') could be also used. However, it contains 179, whereas smart stopword list does 571 words, including ‘i’, ‘me’, ‘my’, ‘myself’, ‘we’, ‘you’, ‘he’, ‘his’, for instance. \n",
    "* stpwrd is here extended with app names that are mentioned in the reviews as well since they are going to be included in every reviews that belong to them."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "a5e2b7d4",
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_stopwords():\n",
    "    stpwrd_url = \"http://www.ai.mit.edu/projects/jmlr/papers/volume5/lewis04a/a11-smart-stop-list/english.stop\"\n",
    "    response = urlopen(stpwrd_url)\n",
    "    stpwrds = response.read().decode('utf-8').replace(\"\\n\", \" \").split()\n",
    "    return stpwrds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "62478259",
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_stopwords(text, stpwrds):\n",
    "    text = text.split(\" \")\n",
    "    words = [w for w in text if w not in stpwrds]\n",
    "    return ' '.join(words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "7749be33",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0       yhr app im littetally addicted things personal...\n",
       "1       tik tok people chance share life stories sad h...\n",
       "2                                    love app im addicted\n",
       "3       app pedophile acts, underage half naked girls ...\n",
       "4       good app dont soo tiktokers committing offense...\n",
       "                              ...                        \n",
       "3096    great app mind ripped driving global warming. ...\n",
       "3097    colleagues wanted car pool respective places l...\n",
       "3098    dark mode rolled everyone. imagine wasting pow...\n",
       "3099    instagram give room share experiences coffee, ...\n",
       "3100    caution app doesnt delete card details beware !!!\n",
       "Name: clean_content, Length: 3101, dtype: object"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stpwrds = generate_stopwords()\n",
    "df['clean_content'] = df['clean_content'].apply(lambda x: remove_stopwords(x, stpwrds))\n",
    "df['clean_content'] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "5972e487",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[\"Calculate more fare than usual and never get any solution of any problem or\\ncouldn't report for driver's bad behavior\"\n",
      " \"calculate fare usual solution problem report driver's bad behavior\"]\n"
     ]
    }
   ],
   "source": [
    "#df['clean_content']\n",
    "print(df.loc[400, ['content','clean_content']].values)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "75a0361f",
   "metadata": {},
   "source": [
    "## Remove punctuation\n",
    "The process of punctuation elimination involves iterating through the series using list comprehension and preserving all elements that do not exist in the __string.punctuation__ list. This list, imported at the beginning using __import string__, comprises all punctuation marks."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "4c665af4",
   "metadata": {},
   "outputs": [],
   "source": [
    "def removePunctuation(text):\n",
    "    no_punc = \"\".join([c for c in text if c not in string.punctuation])\n",
    "    return no_punc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "d6b4e5b1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0       yhr app im littetally addicted things personal...\n",
       "1       tik tok people chance share life stories sad h...\n",
       "2                                    love app im addicted\n",
       "3       app pedophile acts underage half naked girls i...\n",
       "4       good app dont soo tiktokers committing offense...\n",
       "                              ...                        \n",
       "3096    great app mind ripped driving global warming c...\n",
       "3097    colleagues wanted car pool respective places l...\n",
       "3098    dark mode rolled everyone imagine wasting powe...\n",
       "3099    instagram give room share experiences coffee p...\n",
       "3100       caution app doesnt delete card details beware \n",
       "Name: clean_content, Length: 3101, dtype: object"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['clean_content'] = df['clean_content'].apply(lambda x: removePunctuation(x))\n",
    "df['clean_content']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "64cd9c1e",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['clean_content'] = df['clean_content'].apply(lambda x: remove_stopwords(x, stpwrds))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "28600e28",
   "metadata": {},
   "source": [
    "## Tokenizing words\n",
    "\n",
    "* __RegexpTokenizer__ is a function that is used to break down a string into smaller substrings based on a specified regular expression pattern. The selected pattern splits up by spaces that are not attached to a digit as numbers are already cleaned from reviews.\n",
    "* __discard\\_empty__ is set to True. It ensures that any empty tokens produced by the tokenizer are removed from the resulting output. \n",
    "(see in https://www.nltk.org/_modules/nltk/tokenize/regexp.html) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "cc02e1fc",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = RegexpTokenizer(r'\\w+|\\$[\\d\\.]+|\\S+', discard_empty=True)\n",
    "df['clean_content'] = df['clean_content'].apply(lambda x: tokenizer.tokenize(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "8a7497c9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0       [yhr, app, im, littetally, addicted, things, p...\n",
      "1       [tik, tok, people, chance, share, life, storie...\n",
      "2                               [love, app, im, addicted]\n",
      "3       [app, pedophile, acts, underage, half, naked, ...\n",
      "4       [good, app, dont, soo, tiktokers, committing, ...\n",
      "                              ...                        \n",
      "3096    [great, app, mind, ripped, driving, global, wa...\n",
      "3097    [colleagues, wanted, car, pool, respective, pl...\n",
      "3098    [dark, mode, rolled, imagine, wasting, power, ...\n",
      "3099    [instagram, give, room, share, experiences, co...\n",
      "3100    [caution, app, doesnt, delete, card, details, ...\n",
      "Name: clean_content, Length: 3101, dtype: object\n",
      "\n",
      "One particular review:\n",
      "[\"Calculate more fare than usual and never get any solution of any problem or\\ncouldn't report for driver's bad behavior\"\n",
      " list(['calculate', 'fare', 'usual', 'solution', 'problem', 'report', 'drivers', 'bad', 'behavior'])]\n"
     ]
    }
   ],
   "source": [
    "print(df['clean_content'])\n",
    "print(\"\\nOne particular review:\")\n",
    "print(df.loc[400, ['content','clean_content']].values)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f70d3733",
   "metadata": {},
   "source": [
    "## Lemmatizing"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6612c2b6",
   "metadata": {},
   "source": [
    "#### WordNet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "3d51d04e",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_part_of_speech(word):\n",
    "    probable_part_of_speech = wordnet.synsets(word)\n",
    "    pos_counts = Counter()\n",
    "    #pos_counts[\"n\"] = len([item for item in probable_part_of_speech if item.pos() == \"n\"])\n",
    "    pos_counts[\"v\"] = len([item for item in probable_part_of_speech if item.pos() == \"v\"])\n",
    "    #pos_counts[\"n\"] = len([item for item in probable_part_of_speech if item.pos() == \"n\"])\n",
    "    pos_counts[\"a\"] = len([item for item in probable_part_of_speech if item.pos() == \"a\"])  \n",
    "    pos_counts[\"n\"] = len([item for item in probable_part_of_speech if item.pos() == \"n\"])\n",
    "    #pos_counts[\"r\"] = len([item for item in probable_part_of_speech if item.pos() == \"r\"])\n",
    "\n",
    "    most_likely_part_of_speech = pos_counts.most_common(1)[0][0]\n",
    "    return most_likely_part_of_speech"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "1edda441",
   "metadata": {},
   "outputs": [],
   "source": [
    "def word_lemmatizer(text, lemmatizer):\n",
    "    lem_text = [lemmatizer.lemmatize(i, get_part_of_speech(i)) for i in text]\n",
    "    return lem_text\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "261a08f1",
   "metadata": {},
   "outputs": [],
   "source": [
    "wordnetlemma =  WordNetLemmatizer()\n",
    "df['clean_content'] = df['clean_content'].apply(lambda x: word_lemmatizer(x, wordnetlemma))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "574198ad",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                                content  \\\n",
      "0     This is yhr best app ever im littetally addict...   \n",
      "1     like tik tok because it gives people a chance ...   \n",
      "2                      I love this app like im addicted   \n",
      "3     This app allows pedophile acts, underage half ...   \n",
      "4     A very good app but I dont like how theres soo...   \n",
      "...                                                 ...   \n",
      "3096  Great app if you don't mind getting ripped off...   \n",
      "3097  Myself and my colleagues wanted to car pool an...   \n",
      "3098  Its 2020 and dark mode is not even rolled out ...   \n",
      "3099  Instagram give a room to share my experiences ...   \n",
      "3100  CAUTION THIS APP DOESNT ALLOW YOU TO DELETE YO...   \n",
      "\n",
      "                                          clean_content  \n",
      "0     [yhr, app, im, littetally, addict, thing, pers...  \n",
      "1     [tik, tok, people, chance, share, life, story,...  \n",
      "2                               [love, app, im, addict]  \n",
      "3     [app, pedophile, act, underage, half, naked, g...  \n",
      "4     [good, app, dont, soo, tiktokers, commit, offe...  \n",
      "...                                                 ...  \n",
      "3096  [great, app, mind, rip, drive, global, warm, c...  \n",
      "3097  [colleague, want, car, pool, respective, place...  \n",
      "3098  [dark, mode, roll, imagine, waste, power, cont...  \n",
      "3099  [instagram, give, room, share, experience, cof...  \n",
      "3100  [caution, app, doesnt, delete, card, detail, b...  \n",
      "\n",
      "[3101 rows x 2 columns]\n",
      "\n",
      "One particular review:\n",
      "['This is yhr best app ever im littetally addicted there is so many things to do personally im a creator but thr videos are AMAZING i would really reccomend you download😁'\n",
      " list(['yhr', 'app', 'im', 'littetally', 'addict', 'thing', 'personally', 'im', 'creator', 'thr', 'video', 'amaze', 'reccomend', 'download'])]\n"
     ]
    }
   ],
   "source": [
    "print(df[['content','clean_content']])\n",
    "print(\"\\nOne particular review:\")\n",
    "print(df.loc[0, ['content','clean_content']].values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "7ba5782b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0       yhr app im littetally addict thing personally ...\n",
       "1       tik tok people chance share life story sad hap...\n",
       "2                                      love app im addict\n",
       "3       app pedophile act underage half naked girl ina...\n",
       "4       good app dont soo tiktokers commit offense rep...\n",
       "                              ...                        \n",
       "3096    great app mind rip drive global warm carbon em...\n",
       "3097    colleague want car pool respective place long ...\n",
       "3098    dark mode roll imagine waste power contribute ...\n",
       "3099    instagram give room share experience coffee pe...\n",
       "3100         caution app doesnt delete card detail beware\n",
       "Name: clean_content, Length: 3101, dtype: object"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['clean_content'] = [' '.join(x) for x in df['clean_content']]\n",
    "df['clean_content']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "b8df4752",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0                                Addiction (internal)\n",
       "1                                               Other\n",
       "2                                Addiction (internal)\n",
       "3                    inappropriate content (internal)\n",
       "4       accountability  (can be internal or external)\n",
       "                            ...                      \n",
       "3096                                   sustainability\n",
       "3097                                   sustainability\n",
       "3098                                   sustainability\n",
       "3099                                             none\n",
       "3100                                          privacy\n",
       "Name: cat1, Length: 3101, dtype: object"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['cat1']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "65bdd91f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0                   addiction \n",
       "1                        Other\n",
       "2                   addiction \n",
       "3       inappropriate content \n",
       "4             accountability  \n",
       "                 ...          \n",
       "3096            sustainability\n",
       "3097            sustainability\n",
       "3098            sustainability\n",
       "3099                      none\n",
       "3100                   privacy\n",
       "Name: cat1_clean, Length: 3101, dtype: object"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['cat1_clean'] = df['cat1'].apply(lambda x: x.lower())\n",
    "df['cat1_clean'] = df['cat1_clean'].str.extract(r'^(.*?)\\(', expand=True)\n",
    "df['cat1_clean'].fillna(df['cat1'], inplace=True)\n",
    "df['cat1_clean']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "92ae6967",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['addiction', 'Other', 'inappropriate content', 'accountability',\n",
       "       'privacy', 'discrimination', 'spreading false information',\n",
       "       'censorship', 'cyberbullying/toxicity', 'safety', 'identity theft',\n",
       "       'scam', 'harmful advertising', 'Noise', 'transparency',\n",
       "       'accessibility', 'Content theft', 'none', 'sustainability'],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['cat1_clean'] = df['cat1_clean'].str.strip()\n",
    "\n",
    "df['cat1'] = df['cat1_clean']\n",
    "df['cat1'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "1d806c59",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>reviewId</th>\n",
       "      <th>userName</th>\n",
       "      <th>userImage</th>\n",
       "      <th>content</th>\n",
       "      <th>score</th>\n",
       "      <th>thumbsUpCount</th>\n",
       "      <th>reviewCreatedVersion</th>\n",
       "      <th>at</th>\n",
       "      <th>replyContent</th>\n",
       "      <th>repliedAt</th>\n",
       "      <th>app_name</th>\n",
       "      <th>cat1</th>\n",
       "      <th>inex1</th>\n",
       "      <th>clean_content</th>\n",
       "      <th>cat1_clean</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>gp:AOqpTOF5UoM-6ovAjd8ULHKjifvZCeJyoJWi4F_IaPO...</td>\n",
       "      <td>Max Toon</td>\n",
       "      <td>https://play-lh.googleusercontent.com/a-/AOh14...</td>\n",
       "      <td>This is yhr best app ever im littetally addict...</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>18.1.3</td>\n",
       "      <td>2020-12-11 19:51:21</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>tiktok</td>\n",
       "      <td>addiction</td>\n",
       "      <td>internal</td>\n",
       "      <td>yhr app im littetally addict thing personally ...</td>\n",
       "      <td>addiction</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>gp:AOqpTOGZ4VkRpsQa_bVYJdVz65yIOLs5jsENbB_aDJe...</td>\n",
       "      <td>Sandy Mason</td>\n",
       "      <td>https://play-lh.googleusercontent.com/-iFGfy5d...</td>\n",
       "      <td>like tik tok because it gives people a chance ...</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>17.3.4</td>\n",
       "      <td>2020-08-21 21:32:37</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>tiktok</td>\n",
       "      <td>Other</td>\n",
       "      <td>NaN</td>\n",
       "      <td>tik tok people chance share life story sad hap...</td>\n",
       "      <td>Other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>gp:AOqpTOF1KZT5ggeQqGpl62-V6QzBxhROn0eutiZMm9l...</td>\n",
       "      <td>izzyiscool</td>\n",
       "      <td>https://play-lh.googleusercontent.com/a-/AOh14...</td>\n",
       "      <td>I love this app like im addicted</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2020-10-12 1:38:54</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>tiktok</td>\n",
       "      <td>addiction</td>\n",
       "      <td>internal</td>\n",
       "      <td>love app im addict</td>\n",
       "      <td>addiction</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>gp:AOqpTOFRuZB5C5PEpW09xVx3pts_63bcWm9DFf4rajR...</td>\n",
       "      <td>Kristopher Lyons</td>\n",
       "      <td>https://play-lh.googleusercontent.com/a-/AOh14...</td>\n",
       "      <td>This app allows pedophile acts, underage half ...</td>\n",
       "      <td>1</td>\n",
       "      <td>17</td>\n",
       "      <td>17.9.5</td>\n",
       "      <td>2020-12-03 9:55:01</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>tiktok</td>\n",
       "      <td>inappropriate content</td>\n",
       "      <td>internal</td>\n",
       "      <td>app pedophile act underage half naked girl ina...</td>\n",
       "      <td>inappropriate content</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>gp:AOqpTOFf_z_S6J2LZIb70xQY4oWiR19R0HN4oIVV4MW...</td>\n",
       "      <td>Certified Skillz</td>\n",
       "      <td>https://play-lh.googleusercontent.com/a-/AOh14...</td>\n",
       "      <td>A very good app but I dont like how theres soo...</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2020-10-13 1:58:36</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>tiktok</td>\n",
       "      <td>accountability</td>\n",
       "      <td>internal</td>\n",
       "      <td>good app dont soo tiktokers commit offense rep...</td>\n",
       "      <td>accountability</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            reviewId          userName  \\\n",
       "0  gp:AOqpTOF5UoM-6ovAjd8ULHKjifvZCeJyoJWi4F_IaPO...          Max Toon   \n",
       "1  gp:AOqpTOGZ4VkRpsQa_bVYJdVz65yIOLs5jsENbB_aDJe...       Sandy Mason   \n",
       "2  gp:AOqpTOF1KZT5ggeQqGpl62-V6QzBxhROn0eutiZMm9l...        izzyiscool   \n",
       "3  gp:AOqpTOFRuZB5C5PEpW09xVx3pts_63bcWm9DFf4rajR...  Kristopher Lyons   \n",
       "4  gp:AOqpTOFf_z_S6J2LZIb70xQY4oWiR19R0HN4oIVV4MW...  Certified Skillz   \n",
       "\n",
       "                                           userImage  \\\n",
       "0  https://play-lh.googleusercontent.com/a-/AOh14...   \n",
       "1  https://play-lh.googleusercontent.com/-iFGfy5d...   \n",
       "2  https://play-lh.googleusercontent.com/a-/AOh14...   \n",
       "3  https://play-lh.googleusercontent.com/a-/AOh14...   \n",
       "4  https://play-lh.googleusercontent.com/a-/AOh14...   \n",
       "\n",
       "                                             content  score  thumbsUpCount  \\\n",
       "0  This is yhr best app ever im littetally addict...      5              0   \n",
       "1  like tik tok because it gives people a chance ...      5              0   \n",
       "2                   I love this app like im addicted      5              0   \n",
       "3  This app allows pedophile acts, underage half ...      1             17   \n",
       "4  A very good app but I dont like how theres soo...      5              0   \n",
       "\n",
       "  reviewCreatedVersion                   at replyContent repliedAt app_name  \\\n",
       "0               18.1.3  2020-12-11 19:51:21          NaN       NaN   tiktok   \n",
       "1               17.3.4  2020-08-21 21:32:37          NaN       NaN   tiktok   \n",
       "2                  NaN   2020-10-12 1:38:54          NaN       NaN   tiktok   \n",
       "3               17.9.5   2020-12-03 9:55:01          NaN       NaN   tiktok   \n",
       "4                  NaN   2020-10-13 1:58:36          NaN       NaN   tiktok   \n",
       "\n",
       "                    cat1     inex1  \\\n",
       "0              addiction  internal   \n",
       "1                  Other       NaN   \n",
       "2              addiction  internal   \n",
       "3  inappropriate content  internal   \n",
       "4         accountability  internal   \n",
       "\n",
       "                                       clean_content             cat1_clean  \n",
       "0  yhr app im littetally addict thing personally ...              addiction  \n",
       "1  tik tok people chance share life story sad hap...                  Other  \n",
       "2                                 love app im addict              addiction  \n",
       "3  app pedophile act underage half naked girl ina...  inappropriate content  \n",
       "4  good app dont soo tiktokers commit offense rep...         accountability  "
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "2e447442",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Calculate more fare than usual and never get any solution of any problem or\n",
      "couldn't report for driver's bad behavior\n",
      "calculate fare usual solution problem report driver bad behavior\n"
     ]
    }
   ],
   "source": [
    "print(df.loc[400, 'content'])\n",
    "print(df.loc[400, 'clean_content'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "f51449f6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Stored 'df' (DataFrame)\n"
     ]
    }
   ],
   "source": [
    "%store df"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}