Binary_Classification.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "660146ae",
   "metadata": {},
   "source": [
    "## Binary Classification of Ethical Concerns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "8de4e255",
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "a9655368",
   "metadata": {},
   "outputs": [],
   "source": [
    "%store -r df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "9c1c5078",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>reviewId</th>\n",
       "      <th>userName</th>\n",
       "      <th>userImage</th>\n",
       "      <th>content</th>\n",
       "      <th>score</th>\n",
       "      <th>thumbsUpCount</th>\n",
       "      <th>reviewCreatedVersion</th>\n",
       "      <th>at</th>\n",
       "      <th>replyContent</th>\n",
       "      <th>repliedAt</th>\n",
       "      <th>app_name</th>\n",
       "      <th>cat1</th>\n",
       "      <th>inex1</th>\n",
       "      <th>clean_content</th>\n",
       "      <th>cat1_clean</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>gp:AOqpTOF5UoM-6ovAjd8ULHKjifvZCeJyoJWi4F_IaPO...</td>\n",
       "      <td>Max Toon</td>\n",
       "      <td>https://play-lh.googleusercontent.com/a-/AOh14...</td>\n",
       "      <td>This is yhr best app ever im littetally addict...</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>18.1.3</td>\n",
       "      <td>2020-12-11 19:51:21</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>tiktok</td>\n",
       "      <td>addiction</td>\n",
       "      <td>internal</td>\n",
       "      <td>yhr app im littetally addict thing personally ...</td>\n",
       "      <td>addiction</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>gp:AOqpTOGZ4VkRpsQa_bVYJdVz65yIOLs5jsENbB_aDJe...</td>\n",
       "      <td>Sandy Mason</td>\n",
       "      <td>https://play-lh.googleusercontent.com/-iFGfy5d...</td>\n",
       "      <td>like tik tok because it gives people a chance ...</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>17.3.4</td>\n",
       "      <td>2020-08-21 21:32:37</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>tiktok</td>\n",
       "      <td>Other</td>\n",
       "      <td>NaN</td>\n",
       "      <td>tik tok people chance share life story sad hap...</td>\n",
       "      <td>Other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>gp:AOqpTOF1KZT5ggeQqGpl62-V6QzBxhROn0eutiZMm9l...</td>\n",
       "      <td>izzyiscool</td>\n",
       "      <td>https://play-lh.googleusercontent.com/a-/AOh14...</td>\n",
       "      <td>I love this app like im addicted</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2020-10-12 1:38:54</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>tiktok</td>\n",
       "      <td>addiction</td>\n",
       "      <td>internal</td>\n",
       "      <td>love app im addict</td>\n",
       "      <td>addiction</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>gp:AOqpTOFRuZB5C5PEpW09xVx3pts_63bcWm9DFf4rajR...</td>\n",
       "      <td>Kristopher Lyons</td>\n",
       "      <td>https://play-lh.googleusercontent.com/a-/AOh14...</td>\n",
       "      <td>This app allows pedophile acts, underage half ...</td>\n",
       "      <td>1</td>\n",
       "      <td>17</td>\n",
       "      <td>17.9.5</td>\n",
       "      <td>2020-12-03 9:55:01</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>tiktok</td>\n",
       "      <td>inappropriate content</td>\n",
       "      <td>internal</td>\n",
       "      <td>app pedophile act underage half naked girl ina...</td>\n",
       "      <td>inappropriate content</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>gp:AOqpTOFf_z_S6J2LZIb70xQY4oWiR19R0HN4oIVV4MW...</td>\n",
       "      <td>Certified Skillz</td>\n",
       "      <td>https://play-lh.googleusercontent.com/a-/AOh14...</td>\n",
       "      <td>A very good app but I dont like how theres soo...</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2020-10-13 1:58:36</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>tiktok</td>\n",
       "      <td>accountability</td>\n",
       "      <td>internal</td>\n",
       "      <td>good app dont soo tiktokers commit offense rep...</td>\n",
       "      <td>accountability</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            reviewId          userName  \\\n",
       "0  gp:AOqpTOF5UoM-6ovAjd8ULHKjifvZCeJyoJWi4F_IaPO...          Max Toon   \n",
       "1  gp:AOqpTOGZ4VkRpsQa_bVYJdVz65yIOLs5jsENbB_aDJe...       Sandy Mason   \n",
       "2  gp:AOqpTOF1KZT5ggeQqGpl62-V6QzBxhROn0eutiZMm9l...        izzyiscool   \n",
       "3  gp:AOqpTOFRuZB5C5PEpW09xVx3pts_63bcWm9DFf4rajR...  Kristopher Lyons   \n",
       "4  gp:AOqpTOFf_z_S6J2LZIb70xQY4oWiR19R0HN4oIVV4MW...  Certified Skillz   \n",
       "\n",
       "                                           userImage  \\\n",
       "0  https://play-lh.googleusercontent.com/a-/AOh14...   \n",
       "1  https://play-lh.googleusercontent.com/-iFGfy5d...   \n",
       "2  https://play-lh.googleusercontent.com/a-/AOh14...   \n",
       "3  https://play-lh.googleusercontent.com/a-/AOh14...   \n",
       "4  https://play-lh.googleusercontent.com/a-/AOh14...   \n",
       "\n",
       "                                             content  score  thumbsUpCount  \\\n",
       "0  This is yhr best app ever im littetally addict...      5              0   \n",
       "1  like tik tok because it gives people a chance ...      5              0   \n",
       "2                   I love this app like im addicted      5              0   \n",
       "3  This app allows pedophile acts, underage half ...      1             17   \n",
       "4  A very good app but I dont like how theres soo...      5              0   \n",
       "\n",
       "  reviewCreatedVersion                   at replyContent repliedAt app_name  \\\n",
       "0               18.1.3  2020-12-11 19:51:21          NaN       NaN   tiktok   \n",
       "1               17.3.4  2020-08-21 21:32:37          NaN       NaN   tiktok   \n",
       "2                  NaN   2020-10-12 1:38:54          NaN       NaN   tiktok   \n",
       "3               17.9.5   2020-12-03 9:55:01          NaN       NaN   tiktok   \n",
       "4                  NaN   2020-10-13 1:58:36          NaN       NaN   tiktok   \n",
       "\n",
       "                    cat1     inex1  \\\n",
       "0              addiction  internal   \n",
       "1                  Other       NaN   \n",
       "2              addiction  internal   \n",
       "3  inappropriate content  internal   \n",
       "4         accountability  internal   \n",
       "\n",
       "                                       clean_content             cat1_clean  \n",
       "0  yhr app im littetally addict thing personally ...              addiction  \n",
       "1  tik tok people chance share life story sad hap...                  Other  \n",
       "2                                 love app im addict              addiction  \n",
       "3  app pedophile act underage half naked girl ina...  inappropriate content  \n",
       "4  good app dont soo tiktokers commit offense rep...         accountability  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset = df\n",
    "dataset.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "ba352d51",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number of all reviews: 3101\n",
      "number of reviews with concerns: 1920\n"
     ]
    }
   ],
   "source": [
    "df_concern = dataset.query(\"cat1 not in ['Other', 'none', 'Noise']\")\n",
    "print(\"number of all reviews:\", len(dataset))\n",
    "print(\"number of reviews with concerns:\", len(df_concern))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "14c0c58b",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset[\"target\"] = np.where(dataset[\"cat1\"].isin(['Other', 'none', 'Noise']), 0, 1)\n",
    "X = dataset['content']\n",
    "y = dataset['target']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "6daa848a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def factorize_concern(dataset):\n",
    "    dataset['cat1_id'] = dataset['cat1'].factorize()[0]\n",
    "    concern_id_df = dataset[['cat1', 'cat1_id']].drop_duplicates().sort_values('cat1_id')\n",
    "    concern_to_id = dict(concern_id_df.values)\n",
    "    id_to_concern = dict(concern_id_df[['cat1_id', 'cat1']].values)\n",
    "    return dataset, concern_id_df, concern_to_id, id_to_concern"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "436216dc",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "X_train, X_test, y_train, y_test, indices_train, indices_test = train_test_split(X, y, dataset.index, test_size=0.75, random_state=42)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "a9a3d2cd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2326"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "a11bd79e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>reviewId</th>\n",
       "      <th>userName</th>\n",
       "      <th>userImage</th>\n",
       "      <th>content</th>\n",
       "      <th>score</th>\n",
       "      <th>thumbsUpCount</th>\n",
       "      <th>reviewCreatedVersion</th>\n",
       "      <th>at</th>\n",
       "      <th>replyContent</th>\n",
       "      <th>repliedAt</th>\n",
       "      <th>app_name</th>\n",
       "      <th>cat1</th>\n",
       "      <th>inex1</th>\n",
       "      <th>clean_content</th>\n",
       "      <th>cat1_clean</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>gp:AOqpTOF5UoM-6ovAjd8ULHKjifvZCeJyoJWi4F_IaPO...</td>\n",
       "      <td>Max Toon</td>\n",
       "      <td>https://play-lh.googleusercontent.com/a-/AOh14...</td>\n",
       "      <td>This is yhr best app ever im littetally addict...</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>18.1.3</td>\n",
       "      <td>2020-12-11 19:51:21</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>tiktok</td>\n",
       "      <td>addiction</td>\n",
       "      <td>internal</td>\n",
       "      <td>yhr app im littetally addict thing personally ...</td>\n",
       "      <td>addiction</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>gp:AOqpTOGZ4VkRpsQa_bVYJdVz65yIOLs5jsENbB_aDJe...</td>\n",
       "      <td>Sandy Mason</td>\n",
       "      <td>https://play-lh.googleusercontent.com/-iFGfy5d...</td>\n",
       "      <td>like tik tok because it gives people a chance ...</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>17.3.4</td>\n",
       "      <td>2020-08-21 21:32:37</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>tiktok</td>\n",
       "      <td>Other</td>\n",
       "      <td>NaN</td>\n",
       "      <td>tik tok people chance share life story sad hap...</td>\n",
       "      <td>Other</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>gp:AOqpTOF1KZT5ggeQqGpl62-V6QzBxhROn0eutiZMm9l...</td>\n",
       "      <td>izzyiscool</td>\n",
       "      <td>https://play-lh.googleusercontent.com/a-/AOh14...</td>\n",
       "      <td>I love this app like im addicted</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2020-10-12 1:38:54</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>tiktok</td>\n",
       "      <td>addiction</td>\n",
       "      <td>internal</td>\n",
       "      <td>love app im addict</td>\n",
       "      <td>addiction</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>gp:AOqpTOFRuZB5C5PEpW09xVx3pts_63bcWm9DFf4rajR...</td>\n",
       "      <td>Kristopher Lyons</td>\n",
       "      <td>https://play-lh.googleusercontent.com/a-/AOh14...</td>\n",
       "      <td>This app allows pedophile acts, underage half ...</td>\n",
       "      <td>1</td>\n",
       "      <td>17</td>\n",
       "      <td>17.9.5</td>\n",
       "      <td>2020-12-03 9:55:01</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>tiktok</td>\n",
       "      <td>inappropriate content</td>\n",
       "      <td>internal</td>\n",
       "      <td>app pedophile act underage half naked girl ina...</td>\n",
       "      <td>inappropriate content</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>gp:AOqpTOFf_z_S6J2LZIb70xQY4oWiR19R0HN4oIVV4MW...</td>\n",
       "      <td>Certified Skillz</td>\n",
       "      <td>https://play-lh.googleusercontent.com/a-/AOh14...</td>\n",
       "      <td>A very good app but I dont like how theres soo...</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2020-10-13 1:58:36</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>tiktok</td>\n",
       "      <td>accountability</td>\n",
       "      <td>internal</td>\n",
       "      <td>good app dont soo tiktokers commit offense rep...</td>\n",
       "      <td>accountability</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3096</th>\n",
       "      <td>gp:AOqpTOEq_J1k1i2jHYXRaGYBfY4NX7PGO63MvChhgR-...</td>\n",
       "      <td>Federation of Gaia</td>\n",
       "      <td>https://play-lh.googleusercontent.com/a-/AOh14...</td>\n",
       "      <td>Great app if you don't mind getting ripped off...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2020-04-24 16:03:08</td>\n",
       "      <td>Hey there, we're always looking to improve you...</td>\n",
       "      <td>2020-04-25 5:42:34</td>\n",
       "      <td>uber</td>\n",
       "      <td>sustainability</td>\n",
       "      <td>external</td>\n",
       "      <td>great app mind rip drive global warm carbon em...</td>\n",
       "      <td>sustainability</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3097</th>\n",
       "      <td>gp:AOqpTOHLG5pnAGLrH3uwIAsyxS1akNQo-lql0kHPAPL...</td>\n",
       "      <td>Chan dar</td>\n",
       "      <td>https://play-lh.googleusercontent.com/a-/AOh14...</td>\n",
       "      <td>Myself and my colleagues wanted to car pool an...</td>\n",
       "      <td>1</td>\n",
       "      <td>20</td>\n",
       "      <td>3.65.1</td>\n",
       "      <td>2015-08-29 16:09:05</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>uber</td>\n",
       "      <td>sustainability</td>\n",
       "      <td>external</td>\n",
       "      <td>colleague want car pool respective place long ...</td>\n",
       "      <td>sustainability</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3098</th>\n",
       "      <td>gp:AOqpTOFW3kL7vyrV7EpBLJeH9nV_Wz7SmVU5FRaa2_j...</td>\n",
       "      <td>Sarthak Sharma</td>\n",
       "      <td>https://play-lh.googleusercontent.com/a-/AOh14...</td>\n",
       "      <td>Its 2020 and dark mode is not even rolled out ...</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>287.0.0.50.119</td>\n",
       "      <td>2020-09-11 5:36:06</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>facebook</td>\n",
       "      <td>sustainability</td>\n",
       "      <td>external</td>\n",
       "      <td>dark mode roll imagine waste power contribute ...</td>\n",
       "      <td>sustainability</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3099</th>\n",
       "      <td>gp:AOqpTOGtKpptH8IZsiA_0GV3V5hXN8O4QucEWHZ5988...</td>\n",
       "      <td>eko purnomowidi</td>\n",
       "      <td>https://play-lh.googleusercontent.com/a-/AOh14...</td>\n",
       "      <td>Instagram give a room to share my experiences ...</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>166.1.0.42.245</td>\n",
       "      <td>2020-11-12 3:48:05</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>instagram</td>\n",
       "      <td>none</td>\n",
       "      <td>external</td>\n",
       "      <td>instagram give room share experience coffee pe...</td>\n",
       "      <td>none</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3100</th>\n",
       "      <td>gp:AOqpTOFvSj-FFZzy1uftIFaVZNTSPuIc1z3beSjvSCJ...</td>\n",
       "      <td>Moe. S Malek Murtuja</td>\n",
       "      <td>https://play-lh.googleusercontent.com/a-/AOh14...</td>\n",
       "      <td>CAUTION THIS APP DOESNT ALLOW YOU TO DELETE YO...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4.214.10002</td>\n",
       "      <td>2018-07-04 12:34:02</td>\n",
       "      <td>Hi Moe, you can only delete a payment method i...</td>\n",
       "      <td>2018-07-06 7:30:25</td>\n",
       "      <td>uber</td>\n",
       "      <td>privacy</td>\n",
       "      <td>internal</td>\n",
       "      <td>caution app doesnt delete card detail beware</td>\n",
       "      <td>privacy</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3101 rows × 16 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               reviewId              userName  \\\n",
       "0     gp:AOqpTOF5UoM-6ovAjd8ULHKjifvZCeJyoJWi4F_IaPO...              Max Toon   \n",
       "1     gp:AOqpTOGZ4VkRpsQa_bVYJdVz65yIOLs5jsENbB_aDJe...           Sandy Mason   \n",
       "2     gp:AOqpTOF1KZT5ggeQqGpl62-V6QzBxhROn0eutiZMm9l...            izzyiscool   \n",
       "3     gp:AOqpTOFRuZB5C5PEpW09xVx3pts_63bcWm9DFf4rajR...      Kristopher Lyons   \n",
       "4     gp:AOqpTOFf_z_S6J2LZIb70xQY4oWiR19R0HN4oIVV4MW...      Certified Skillz   \n",
       "...                                                 ...                   ...   \n",
       "3096  gp:AOqpTOEq_J1k1i2jHYXRaGYBfY4NX7PGO63MvChhgR-...    Federation of Gaia   \n",
       "3097  gp:AOqpTOHLG5pnAGLrH3uwIAsyxS1akNQo-lql0kHPAPL...              Chan dar   \n",
       "3098  gp:AOqpTOFW3kL7vyrV7EpBLJeH9nV_Wz7SmVU5FRaa2_j...        Sarthak Sharma   \n",
       "3099  gp:AOqpTOGtKpptH8IZsiA_0GV3V5hXN8O4QucEWHZ5988...       eko purnomowidi   \n",
       "3100  gp:AOqpTOFvSj-FFZzy1uftIFaVZNTSPuIc1z3beSjvSCJ...  Moe. S Malek Murtuja   \n",
       "\n",
       "                                              userImage  \\\n",
       "0     https://play-lh.googleusercontent.com/a-/AOh14...   \n",
       "1     https://play-lh.googleusercontent.com/-iFGfy5d...   \n",
       "2     https://play-lh.googleusercontent.com/a-/AOh14...   \n",
       "3     https://play-lh.googleusercontent.com/a-/AOh14...   \n",
       "4     https://play-lh.googleusercontent.com/a-/AOh14...   \n",
       "...                                                 ...   \n",
       "3096  https://play-lh.googleusercontent.com/a-/AOh14...   \n",
       "3097  https://play-lh.googleusercontent.com/a-/AOh14...   \n",
       "3098  https://play-lh.googleusercontent.com/a-/AOh14...   \n",
       "3099  https://play-lh.googleusercontent.com/a-/AOh14...   \n",
       "3100  https://play-lh.googleusercontent.com/a-/AOh14...   \n",
       "\n",
       "                                                content  score  thumbsUpCount  \\\n",
       "0     This is yhr best app ever im littetally addict...      5              0   \n",
       "1     like tik tok because it gives people a chance ...      5              0   \n",
       "2                      I love this app like im addicted      5              0   \n",
       "3     This app allows pedophile acts, underage half ...      1             17   \n",
       "4     A very good app but I dont like how theres soo...      5              0   \n",
       "...                                                 ...    ...            ...   \n",
       "3096  Great app if you don't mind getting ripped off...      1              0   \n",
       "3097  Myself and my colleagues wanted to car pool an...      1             20   \n",
       "3098  Its 2020 and dark mode is not even rolled out ...      1              2   \n",
       "3099  Instagram give a room to share my experiences ...      5              0   \n",
       "3100  CAUTION THIS APP DOESNT ALLOW YOU TO DELETE YO...      1              1   \n",
       "\n",
       "     reviewCreatedVersion                   at  \\\n",
       "0                  18.1.3  2020-12-11 19:51:21   \n",
       "1                  17.3.4  2020-08-21 21:32:37   \n",
       "2                     NaN   2020-10-12 1:38:54   \n",
       "3                  17.9.5   2020-12-03 9:55:01   \n",
       "4                     NaN   2020-10-13 1:58:36   \n",
       "...                   ...                  ...   \n",
       "3096                  NaN  2020-04-24 16:03:08   \n",
       "3097               3.65.1  2015-08-29 16:09:05   \n",
       "3098       287.0.0.50.119   2020-09-11 5:36:06   \n",
       "3099       166.1.0.42.245   2020-11-12 3:48:05   \n",
       "3100          4.214.10002  2018-07-04 12:34:02   \n",
       "\n",
       "                                           replyContent           repliedAt  \\\n",
       "0                                                   NaN                 NaN   \n",
       "1                                                   NaN                 NaN   \n",
       "2                                                   NaN                 NaN   \n",
       "3                                                   NaN                 NaN   \n",
       "4                                                   NaN                 NaN   \n",
       "...                                                 ...                 ...   \n",
       "3096  Hey there, we're always looking to improve you...  2020-04-25 5:42:34   \n",
       "3097                                                NaN                 NaN   \n",
       "3098                                                NaN                 NaN   \n",
       "3099                                                NaN                 NaN   \n",
       "3100  Hi Moe, you can only delete a payment method i...  2018-07-06 7:30:25   \n",
       "\n",
       "       app_name                   cat1     inex1  \\\n",
       "0        tiktok              addiction  internal   \n",
       "1        tiktok                  Other       NaN   \n",
       "2        tiktok              addiction  internal   \n",
       "3        tiktok  inappropriate content  internal   \n",
       "4        tiktok         accountability  internal   \n",
       "...         ...                    ...       ...   \n",
       "3096       uber         sustainability  external   \n",
       "3097       uber         sustainability  external   \n",
       "3098   facebook         sustainability  external   \n",
       "3099  instagram                   none  external   \n",
       "3100       uber                privacy  internal   \n",
       "\n",
       "                                          clean_content  \\\n",
       "0     yhr app im littetally addict thing personally ...   \n",
       "1     tik tok people chance share life story sad hap...   \n",
       "2                                    love app im addict   \n",
       "3     app pedophile act underage half naked girl ina...   \n",
       "4     good app dont soo tiktokers commit offense rep...   \n",
       "...                                                 ...   \n",
       "3096  great app mind rip drive global warm carbon em...   \n",
       "3097  colleague want car pool respective place long ...   \n",
       "3098  dark mode roll imagine waste power contribute ...   \n",
       "3099  instagram give room share experience coffee pe...   \n",
       "3100       caution app doesnt delete card detail beware   \n",
       "\n",
       "                 cat1_clean  target  \n",
       "0                 addiction       1  \n",
       "1                     Other       0  \n",
       "2                 addiction       1  \n",
       "3     inappropriate content       1  \n",
       "4            accountability       1  \n",
       "...                     ...     ...  \n",
       "3096         sustainability       1  \n",
       "3097         sustainability       1  \n",
       "3098         sustainability       1  \n",
       "3099                   none       0  \n",
       "3100                privacy       1  \n",
       "\n",
       "[3101 rows x 16 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "2035690f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Beginning with Default Vectorizers\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "tfidf = TfidfVectorizer()\n",
    "# Setting up fidf X data\n",
    "X_train_tfidf = tfidf.fit_transform(X_train).toarray()\n",
    "X_test_tfidf = tfidf.transform(X_test).toarray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "3211db66",
   "metadata": {},
   "outputs": [],
   "source": [
    "models = {}\n",
    "\n",
    "# Random Forest\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "models['Random Forest'] = RandomForestClassifier()\n",
    "\n",
    "# Support Vector Machines\n",
    "from sklearn.svm import LinearSVC\n",
    "models['Support Vector Machines'] = LinearSVC()\n",
    "\n",
    "# Naive Bayes\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "models['Naive Bayes'] = MultinomialNB(alpha=0.4, fit_prior=False)\n",
    "\n",
    "# Logistic Regression\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "models['Logistic Regression'] = LogisticRegression()\n",
    "\n",
    "# MLP Classifier\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "models['MLP']  = MLPClassifier(\n",
    "                        hidden_layer_sizes=(15,),\n",
    "                        random_state=5,\n",
    "#                         max_iter=3000,\n",
    "                        alpha=0.0008\n",
    "                       )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "b88f9f02",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score\n",
    "\n",
    "accuracy, f1score, precision, recall = {}, {}, {}, {}\n",
    "\n",
    "for key in models.keys():\n",
    "    \n",
    "    # Fit the classifier\n",
    "    models[key].fit(X_train_tfidf, y_train)\n",
    "    \n",
    "    # Make predictions\n",
    "    predictions = models[key].predict(X_test_tfidf)\n",
    "    \n",
    "    # Calculate metrics\n",
    "    accuracy[key] = accuracy_score(predictions, y_test)\n",
    "    precision[key] = precision_score(predictions, y_test)\n",
    "    recall[key] = recall_score(predictions, y_test)\n",
    "    f1score[key] = f1_score(predictions, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "7e0d585e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Recall</th>\n",
       "      <th>F1-score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Random Forest</th>\n",
       "      <td>0.809974</td>\n",
       "      <td>0.903005</td>\n",
       "      <td>0.815043</td>\n",
       "      <td>0.856773</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Support Vector Machines</th>\n",
       "      <td>0.852537</td>\n",
       "      <td>0.896175</td>\n",
       "      <td>0.872921</td>\n",
       "      <td>0.884395</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Naive Bayes</th>\n",
       "      <td>0.864144</td>\n",
       "      <td>0.923497</td>\n",
       "      <td>0.868895</td>\n",
       "      <td>0.895364</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Logistic Regression</th>\n",
       "      <td>0.828891</td>\n",
       "      <td>0.946721</td>\n",
       "      <td>0.812427</td>\n",
       "      <td>0.874448</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MLP</th>\n",
       "      <td>0.854256</td>\n",
       "      <td>0.887295</td>\n",
       "      <td>0.881874</td>\n",
       "      <td>0.884576</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                         Accuracy  Precision    Recall  F1-score\n",
       "Random Forest            0.809974   0.903005  0.815043  0.856773\n",
       "Support Vector Machines  0.852537   0.896175  0.872921  0.884395\n",
       "Naive Bayes              0.864144   0.923497  0.868895  0.895364\n",
       "Logistic Regression      0.828891   0.946721  0.812427  0.874448\n",
       "MLP                      0.854256   0.887295  0.881874  0.884576"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# With all metrics stored, we can use pandas to view the data as a table:\n",
    "\n",
    "df_model = pd.DataFrame(index=models.keys(), columns=['Accuracy', 'Precision', 'Recall', 'F1-score'])\n",
    "df_model['Accuracy'] = accuracy.values()\n",
    "df_model['Precision'] = precision.values()\n",
    "df_model['Recall'] = recall.values()\n",
    "df_model['F1-score'] = f1score.values()\n",
    "df_model.to_csv('binary_classification_results.csv')\n",
    "df_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "7daee863",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 1, 0, ..., 1, 1, 0])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rf = models['Naive Bayes']\n",
    "preds = rf.predict(X_test_tfidf)\n",
    "preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "64342d1f",
   "metadata": {},
   "outputs": [],
   "source": [
    "def add_predict(df, preds, indices_test):\n",
    "    for ind, pred in zip(indices_test, preds):\n",
    "        df.at[ind, \"predicted_bc\"] = int(pred)\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "5b7e43fd",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_pred_bc = add_predict(dataset, preds, indices_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "31739b38",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1., 0.])"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset_pred_bc = dataset_pred_bc[pd.notnull(dataset_pred_bc['predicted_bc'])]\n",
    "dataset_pred_bc['predicted_bc'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "6aaf04dd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['addiction', 'inappropriate content', 'Other', 'privacy',\n",
       "       'discrimination', 'spreading false information', 'censorship',\n",
       "       'safety', 'identity theft', 'cyberbullying/toxicity', 'scam',\n",
       "       'harmful advertising', 'Noise', 'accountability', 'transparency',\n",
       "       'accessibility', 'Content theft', 'sustainability', 'none'],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset_pred_bc = dataset_pred_bc[dataset_pred_bc['predicted_bc']==1]\n",
    "dataset_pred_bc['cat1'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "fe3a8b65",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Stored 'dataset_pred_bc' (DataFrame)\n"
     ]
    }
   ],
   "source": [
    "%store dataset_pred_bc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "70514918",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cat1</th>\n",
       "      <th>target</th>\n",
       "      <th>counts</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Content theft</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Noise</td>\n",
       "      <td>0</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Other</td>\n",
       "      <td>0</td>\n",
       "      <td>181</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>accessibility</td>\n",
       "      <td>1</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>accountability</td>\n",
       "      <td>1</td>\n",
       "      <td>103</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>addiction</td>\n",
       "      <td>1</td>\n",
       "      <td>79</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>censorship</td>\n",
       "      <td>1</td>\n",
       "      <td>132</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>cyberbullying/toxicity</td>\n",
       "      <td>1</td>\n",
       "      <td>103</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>discrimination</td>\n",
       "      <td>1</td>\n",
       "      <td>209</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>harmful advertising</td>\n",
       "      <td>1</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>identity theft</td>\n",
       "      <td>1</td>\n",
       "      <td>75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>inappropriate content</td>\n",
       "      <td>1</td>\n",
       "      <td>157</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>none</td>\n",
       "      <td>0</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>privacy</td>\n",
       "      <td>1</td>\n",
       "      <td>107</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>safety</td>\n",
       "      <td>1</td>\n",
       "      <td>103</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>scam</td>\n",
       "      <td>1</td>\n",
       "      <td>113</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>spreading false information</td>\n",
       "      <td>1</td>\n",
       "      <td>81</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>sustainability</td>\n",
       "      <td>1</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>transparency</td>\n",
       "      <td>1</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                           cat1  target  counts\n",
       "0                 Content theft       1       6\n",
       "1                         Noise       0      11\n",
       "2                         Other       0     181\n",
       "3                 accessibility       1      20\n",
       "4                accountability       1     103\n",
       "5                     addiction       1      79\n",
       "6                    censorship       1     132\n",
       "7        cyberbullying/toxicity       1     103\n",
       "8                discrimination       1     209\n",
       "9           harmful advertising       1      32\n",
       "10               identity theft       1      75\n",
       "11        inappropriate content       1     157\n",
       "12                         none       0      12\n",
       "13                      privacy       1     107\n",
       "14                       safety       1     103\n",
       "15                         scam       1     113\n",
       "16  spreading false information       1      81\n",
       "17               sustainability       1       8\n",
       "18                 transparency       1      24"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_count = dataset_pred_bc.groupby(['cat1', 'target']).clean_content.count().reset_index(name='counts')\n",
    "df_count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "360ee3fd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1556\n",
      "2326\n"
     ]
    }
   ],
   "source": [
    "print(len(dataset_pred_bc[dataset_pred_bc['predicted_bc']==1]))\n",
    "print(len(X_test_tfidf))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f1e9f1e5",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}