26--Classify PublicSphere-benchmark.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "initial_id",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-05-15T14:55:04.880562Z",
     "start_time": "2024-05-15T14:55:04.876394Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "A module that was compiled using NumPy 1.x cannot be run in\n",
      "NumPy 2.0.2 as it may crash. To support both 1.x and 2.x\n",
      "versions of NumPy, modules must be compiled with NumPy 2.0.\n",
      "Some module may need to rebuild instead e.g. with 'pybind11>=2.12'.\n",
      "\n",
      "If you are a user of the module, the easiest solution will be to\n",
      "downgrade to 'numpy<2' or try to upgrade the affected module.\n",
      "We expect that some modules will need time to support NumPy 2.\n",
      "\n",
      "Traceback (most recent call last):  File \"<frozen runpy>\", line 198, in _run_module_as_main\n",
      "  File \"<frozen runpy>\", line 88, in _run_code\n",
      "  File \"c:\\Users\\sstolwi\\Github\\llmdiv\\Lib\\site-packages\\ipykernel_launcher.py\", line 18, in <module>\n",
      "    app.launch_new_instance()\n",
      "  File \"c:\\Users\\sstolwi\\Github\\llmdiv\\Lib\\site-packages\\traitlets\\config\\application.py\", line 1075, in launch_instance\n",
      "    app.start()\n",
      "  File \"c:\\Users\\sstolwi\\Github\\llmdiv\\Lib\\site-packages\\ipykernel\\kernelapp.py\", line 739, in start\n",
      "    self.io_loop.start()\n",
      "  File \"c:\\Users\\sstolwi\\Github\\llmdiv\\Lib\\site-packages\\tornado\\platform\\asyncio.py\", line 205, in start\n",
      "    self.asyncio_loop.run_forever()\n",
      "  File \"C:\\Program Files\\WindowsApps\\PythonSoftwareFoundation.Python.3.12_3.12.2288.0_x64__qbz5n2kfra8p0\\Lib\\asyncio\\base_events.py\", line 640, in run_forever\n",
      "    self._run_once()\n",
      "  File \"C:\\Program Files\\WindowsApps\\PythonSoftwareFoundation.Python.3.12_3.12.2288.0_x64__qbz5n2kfra8p0\\Lib\\asyncio\\base_events.py\", line 1992, in _run_once\n",
      "    handle._run()\n",
      "  File \"C:\\Program Files\\WindowsApps\\PythonSoftwareFoundation.Python.3.12_3.12.2288.0_x64__qbz5n2kfra8p0\\Lib\\asyncio\\events.py\", line 88, in _run\n",
      "    self._context.run(self._callback, *self._args)\n",
      "  File \"c:\\Users\\sstolwi\\Github\\llmdiv\\Lib\\site-packages\\ipykernel\\kernelbase.py\", line 545, in dispatch_queue\n",
      "    await self.process_one()\n",
      "  File \"c:\\Users\\sstolwi\\Github\\llmdiv\\Lib\\site-packages\\ipykernel\\kernelbase.py\", line 534, in process_one\n",
      "    await dispatch(*args)\n",
      "  File \"c:\\Users\\sstolwi\\Github\\llmdiv\\Lib\\site-packages\\ipykernel\\kernelbase.py\", line 437, in dispatch_shell\n",
      "    await result\n",
      "  File \"c:\\Users\\sstolwi\\Github\\llmdiv\\Lib\\site-packages\\ipykernel\\ipkernel.py\", line 362, in execute_request\n",
      "    await super().execute_request(stream, ident, parent)\n",
      "  File \"c:\\Users\\sstolwi\\Github\\llmdiv\\Lib\\site-packages\\ipykernel\\kernelbase.py\", line 778, in execute_request\n",
      "    reply_content = await reply_content\n",
      "  File \"c:\\Users\\sstolwi\\Github\\llmdiv\\Lib\\site-packages\\ipykernel\\ipkernel.py\", line 449, in do_execute\n",
      "    res = shell.run_cell(\n",
      "  File \"c:\\Users\\sstolwi\\Github\\llmdiv\\Lib\\site-packages\\ipykernel\\zmqshell.py\", line 549, in run_cell\n",
      "    return super().run_cell(*args, **kwargs)\n",
      "  File \"c:\\Users\\sstolwi\\Github\\llmdiv\\Lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3075, in run_cell\n",
      "    result = self._run_cell(\n",
      "  File \"c:\\Users\\sstolwi\\Github\\llmdiv\\Lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3130, in _run_cell\n",
      "    result = runner(coro)\n",
      "  File \"c:\\Users\\sstolwi\\Github\\llmdiv\\Lib\\site-packages\\IPython\\core\\async_helpers.py\", line 128, in _pseudo_sync_runner\n",
      "    coro.send(None)\n",
      "  File \"c:\\Users\\sstolwi\\Github\\llmdiv\\Lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3334, in run_cell_async\n",
      "    has_raised = await self.run_ast_nodes(code_ast.body, cell_name,\n",
      "  File \"c:\\Users\\sstolwi\\Github\\llmdiv\\Lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3517, in run_ast_nodes\n",
      "    if await self.run_code(code, result, async_=asy):\n",
      "  File \"c:\\Users\\sstolwi\\Github\\llmdiv\\Lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3577, in run_code\n",
      "    exec(code_obj, self.user_global_ns, self.user_ns)\n",
      "  File \"C:\\Users\\sstolwi\\AppData\\Local\\Temp\\ipykernel_23080\\266960439.py\", line 6, in <module>\n",
      "    import src\n",
      "  File \"c:\\Users\\sstolwi\\Github\\TWON-Metrics\\src\\__init__.py\", line 1, in <module>\n",
      "    from .hf_classify import HFClassify\n",
      "  File \"c:\\Users\\sstolwi\\Github\\TWON-Metrics\\src\\hf_classify.py\", line 5, in <module>\n",
      "    import torch\n",
      "  File \"c:\\Users\\sstolwi\\Github\\llmdiv\\Lib\\site-packages\\torch\\__init__.py\", line 1477, in <module>\n",
      "    from .functional import *  # noqa: F403\n",
      "  File \"c:\\Users\\sstolwi\\Github\\llmdiv\\Lib\\site-packages\\torch\\functional.py\", line 9, in <module>\n",
      "    import torch.nn.functional as F\n",
      "  File \"c:\\Users\\sstolwi\\Github\\llmdiv\\Lib\\site-packages\\torch\\nn\\__init__.py\", line 1, in <module>\n",
      "    from .modules import *  # noqa: F403\n",
      "  File \"c:\\Users\\sstolwi\\Github\\llmdiv\\Lib\\site-packages\\torch\\nn\\modules\\__init__.py\", line 35, in <module>\n",
      "    from .transformer import TransformerEncoder, TransformerDecoder, \\\n",
      "  File \"c:\\Users\\sstolwi\\Github\\llmdiv\\Lib\\site-packages\\torch\\nn\\modules\\transformer.py\", line 20, in <module>\n",
      "    device: torch.device = torch.device(torch._C._get_default_device()),  # torch.device('cpu'),\n",
      "c:\\Users\\sstolwi\\Github\\llmdiv\\Lib\\site-packages\\torch\\nn\\modules\\transformer.py:20: UserWarning: Failed to initialize NumPy: _ARRAY_API not found (Triggered internally at ..\\torch\\csrc\\utils\\tensor_numpy.cpp:84.)\n",
      "  device: torch.device = torch.device(torch._C._get_default_device()),  # torch.device('cpu'),\n",
      "c:\\Users\\sstolwi\\Github\\llmdiv\\Lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "import typing\n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "import config\n",
    "import src\n",
    "import requests\n",
    "import tqdm\n",
    "import json\n",
    "import numpy as np\n",
    "\n",
    "from sklearn.metrics import cohen_kappa_score, classification_report\n",
    "import krippendorff\n",
    "import yaml\n",
    "\n",
    "import os\n",
    "from dotenv import load_dotenv\n",
    "load_dotenv(\"sjoerdAzure.env\")  # Load environment variables from .env file\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "3d025f8b6b833773",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-05-15T14:55:04.892387Z",
     "start_time": "2024-05-15T14:55:04.882915Z"
    }
   },
   "outputs": [],
   "source": [
    "CFG = config.Config()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "8ad2aee526efc24d",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-05-15T14:55:04.913299Z",
     "start_time": "2024-05-15T14:55:04.894258Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>StartDate</th>\n",
       "      <th>RecordedDate</th>\n",
       "      <th>IPAddress</th>\n",
       "      <th>Finished</th>\n",
       "      <th>Coder</th>\n",
       "      <th>ID</th>\n",
       "      <th>Mark_ID</th>\n",
       "      <th>Genre</th>\n",
       "      <th>topiccode</th>\n",
       "      <th>Platform</th>\n",
       "      <th>...</th>\n",
       "      <th>dislikeCount_video</th>\n",
       "      <th>likeCount_video</th>\n",
       "      <th>date_difference</th>\n",
       "      <th>commentCount_video</th>\n",
       "      <th>replyCount_comment</th>\n",
       "      <th>topic</th>\n",
       "      <th>subscribers</th>\n",
       "      <th>HATELIST_FOCUSED_DUMMY</th>\n",
       "      <th>Time_comment_year</th>\n",
       "      <th>Time_video_year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5/30/2021 13:03:17</td>\n",
       "      <td>5/30/2021 13:04:17</td>\n",
       "      <td>62.194.51.29</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>UgyPHwv8G0cDE6-wEgl4AaABAg.8_0ZjJKSJty8_0kXGkAd2U</td>\n",
       "      <td>119</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>2017</td>\n",
       "      <td>2017.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>10/11/2021 10:34:05</td>\n",
       "      <td>10/11/2021 10:36:46</td>\n",
       "      <td>213.127.109.191</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>Ugx2WXq9UdV8mPPjejJ4AaABAg.8yHCKV0Boe58yYRxEQEF45</td>\n",
       "      <td>282</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>195.0</td>\n",
       "      <td>3817.0</td>\n",
       "      <td>743.0</td>\n",
       "      <td>1748.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>economy</td>\n",
       "      <td>3630000.0</td>\n",
       "      <td>0</td>\n",
       "      <td>2019</td>\n",
       "      <td>2019.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>9/9/2021 18:49:48</td>\n",
       "      <td>9/9/2021 18:51:32</td>\n",
       "      <td>213.127.110.0</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>1110578710648890000</td>\n",
       "      <td>372</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>2019</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6/6/2021 16:12:46</td>\n",
       "      <td>6/6/2021 16:16:16</td>\n",
       "      <td>213.127.76.145</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>UgwUPFScjJ0MCeaP2F54AaABAg.8lvp3fc9Euf8lvvgsUgEgV</td>\n",
       "      <td>769</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>2018</td>\n",
       "      <td>2018.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6/13/2021 13:25:49</td>\n",
       "      <td>6/13/2021 13:27:28</td>\n",
       "      <td>213.127.82.232</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>UgwWKCWtSJdFvjGHvTp4AaABAg.8kUC5dGrQ2H8kUDRihE2f3</td>\n",
       "      <td>1206</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>2018</td>\n",
       "      <td>2018.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3857</th>\n",
       "      <td>8/19/2021 14:50:13</td>\n",
       "      <td>8/19/2021 14:54:28</td>\n",
       "      <td>62.194.51.29</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>1152219467579100000</td>\n",
       "      <td>10000695</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>2019</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3858</th>\n",
       "      <td>8/19/2021 15:10:27</td>\n",
       "      <td>8/19/2021 15:12:21</td>\n",
       "      <td>62.194.51.29</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>1085362296472430000</td>\n",
       "      <td>10007008</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>2019</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3859</th>\n",
       "      <td>10/6/2021 16:08:39</td>\n",
       "      <td>10/6/2021 16:10:42</td>\n",
       "      <td>213.127.113.113</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>UghFY3QJ6nmT_ngCoAEC.7-H0Z7--wxd8goqpaPs-bl</td>\n",
       "      <td>20000102</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>2820.0</td>\n",
       "      <td>12475.0</td>\n",
       "      <td>3803.0</td>\n",
       "      <td>4785.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>east</td>\n",
       "      <td>6740000.0</td>\n",
       "      <td>0</td>\n",
       "      <td>2018</td>\n",
       "      <td>2010.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3860</th>\n",
       "      <td>10/15/2021 18:30:04</td>\n",
       "      <td>10/15/2021 18:35:40</td>\n",
       "      <td>213.127.109.191</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>UgyWabsmmnq3zam4DgZ4AaABAg</td>\n",
       "      <td>20000418</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>118.0</td>\n",
       "      <td>31761.0</td>\n",
       "      <td>1531.0</td>\n",
       "      <td>2206.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>east</td>\n",
       "      <td>6800000.0</td>\n",
       "      <td>0</td>\n",
       "      <td>2018</td>\n",
       "      <td>2015.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3861</th>\n",
       "      <td>11/19/2021 17:49:17</td>\n",
       "      <td>11/19/2021 17:51:04</td>\n",
       "      <td>213.127.109.191</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>UgwPOHIDyICm10k0Mvx4AaABAg</td>\n",
       "      <td>20001003</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1358.0</td>\n",
       "      <td>5740.0</td>\n",
       "      <td>2276.0</td>\n",
       "      <td>2887.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>east</td>\n",
       "      <td>549000.0</td>\n",
       "      <td>0</td>\n",
       "      <td>2018</td>\n",
       "      <td>2018.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3862 rows × 79 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                StartDate         RecordedDate        IPAddress  Finished  \\\n",
       "0      5/30/2021 13:03:17   5/30/2021 13:04:17     62.194.51.29         1   \n",
       "1     10/11/2021 10:34:05  10/11/2021 10:36:46  213.127.109.191         1   \n",
       "2       9/9/2021 18:49:48    9/9/2021 18:51:32    213.127.110.0         1   \n",
       "3       6/6/2021 16:12:46    6/6/2021 16:16:16   213.127.76.145         1   \n",
       "4      6/13/2021 13:25:49   6/13/2021 13:27:28   213.127.82.232         1   \n",
       "...                   ...                  ...              ...       ...   \n",
       "3857   8/19/2021 14:50:13   8/19/2021 14:54:28     62.194.51.29         1   \n",
       "3858   8/19/2021 15:10:27   8/19/2021 15:12:21     62.194.51.29         1   \n",
       "3859   10/6/2021 16:08:39   10/6/2021 16:10:42  213.127.113.113         1   \n",
       "3860  10/15/2021 18:30:04  10/15/2021 18:35:40  213.127.109.191         1   \n",
       "3861  11/19/2021 17:49:17  11/19/2021 17:51:04  213.127.109.191         1   \n",
       "\n",
       "      Coder                                                 ID   Mark_ID  \\\n",
       "0         6  UgyPHwv8G0cDE6-wEgl4AaABAg.8_0ZjJKSJty8_0kXGkAd2U       119   \n",
       "1         6  Ugx2WXq9UdV8mPPjejJ4AaABAg.8yHCKV0Boe58yYRxEQEF45       282   \n",
       "2         6                                1110578710648890000       372   \n",
       "3         6  UgwUPFScjJ0MCeaP2F54AaABAg.8lvp3fc9Euf8lvvgsUgEgV       769   \n",
       "4         6  UgwWKCWtSJdFvjGHvTp4AaABAg.8kUC5dGrQ2H8kUDRihE2f3      1206   \n",
       "...     ...                                                ...       ...   \n",
       "3857      6                                1152219467579100000  10000695   \n",
       "3858      6                                1085362296472430000  10007008   \n",
       "3859      6        UghFY3QJ6nmT_ngCoAEC.7-H0Z7--wxd8goqpaPs-bl  20000102   \n",
       "3860      6                         UgyWabsmmnq3zam4DgZ4AaABAg  20000418   \n",
       "3861      6                         UgwPOHIDyICm10k0Mvx4AaABAg  20001003   \n",
       "\n",
       "      Genre  topiccode  Platform  ...  dislikeCount_video likeCount_video  \\\n",
       "0         0          0         1  ...                 NaN             NaN   \n",
       "1         1          2         1  ...               195.0          3817.0   \n",
       "2         2          4         2  ...                 NaN             NaN   \n",
       "3         0          0         1  ...                 NaN             NaN   \n",
       "4         0          0         1  ...                 NaN             NaN   \n",
       "...     ...        ...       ...  ...                 ...             ...   \n",
       "3857      0          4         2  ...                 NaN             NaN   \n",
       "3858      1          4         2  ...                 NaN             NaN   \n",
       "3859      0          3         1  ...              2820.0         12475.0   \n",
       "3860      2          3         1  ...               118.0         31761.0   \n",
       "3861      0          3         1  ...              1358.0          5740.0   \n",
       "\n",
       "      date_difference  commentCount_video  replyCount_comment    topic  \\\n",
       "0                 NaN                 NaN                 NaN      NaN   \n",
       "1               743.0              1748.0                 NaN  economy   \n",
       "2                 NaN                 NaN                 NaN      NaN   \n",
       "3                 NaN                 NaN                 NaN      NaN   \n",
       "4                 NaN                 NaN                 NaN      NaN   \n",
       "...               ...                 ...                 ...      ...   \n",
       "3857              NaN                 NaN                 NaN      NaN   \n",
       "3858              NaN                 NaN                 NaN      NaN   \n",
       "3859           3803.0              4785.0                 NaN     east   \n",
       "3860           1531.0              2206.0                 0.0     east   \n",
       "3861           2276.0              2887.0                 1.0     east   \n",
       "\n",
       "      subscribers  HATELIST_FOCUSED_DUMMY  Time_comment_year Time_video_year  \n",
       "0             NaN                       0               2017          2017.0  \n",
       "1       3630000.0                       0               2019          2019.0  \n",
       "2             NaN                       0               2019             NaN  \n",
       "3             NaN                       0               2018          2018.0  \n",
       "4             NaN                       0               2018          2018.0  \n",
       "...           ...                     ...                ...             ...  \n",
       "3857          NaN                       0               2019             NaN  \n",
       "3858          NaN                       0               2019             NaN  \n",
       "3859    6740000.0                       0               2018          2010.0  \n",
       "3860    6800000.0                       0               2018          2015.0  \n",
       "3861     549000.0                       0               2018          2018.0  \n",
       "\n",
       "[3862 rows x 79 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset: pd.DataFrame = pd.read_csv('data/publicsphere/full_data.csv')\n",
    "dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "b12e33b5",
   "metadata": {},
   "outputs": [],
   "source": [
    "#load train/test data\n",
    "trainset: pd.DataFrame = pd.read_csv('data/publicsphere/train.csv')\n",
    "testset: pd.DataFrame = pd.read_csv('data/publicsphere/test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "30b31721",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>StartDate</th>\n",
       "      <th>RecordedDate</th>\n",
       "      <th>IPAddress</th>\n",
       "      <th>Finished</th>\n",
       "      <th>Coder</th>\n",
       "      <th>ID</th>\n",
       "      <th>Mark_ID</th>\n",
       "      <th>Genre</th>\n",
       "      <th>topiccode</th>\n",
       "      <th>Platform</th>\n",
       "      <th>...</th>\n",
       "      <th>dislikeCount_video</th>\n",
       "      <th>likeCount_video</th>\n",
       "      <th>date_difference</th>\n",
       "      <th>commentCount_video</th>\n",
       "      <th>replyCount_comment</th>\n",
       "      <th>topic</th>\n",
       "      <th>subscribers</th>\n",
       "      <th>HATELIST_FOCUSED_DUMMY</th>\n",
       "      <th>Time_comment_year</th>\n",
       "      <th>Time_video_year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5/30/2021 13:03:17</td>\n",
       "      <td>5/30/2021 13:04:17</td>\n",
       "      <td>62.194.51.29</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>UgyPHwv8G0cDE6-wEgl4AaABAg.8_0ZjJKSJty8_0kXGkAd2U</td>\n",
       "      <td>119</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>2017</td>\n",
       "      <td>2017.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9/9/2021 18:49:48</td>\n",
       "      <td>9/9/2021 18:51:32</td>\n",
       "      <td>213.127.110.0</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>1110578710648890000</td>\n",
       "      <td>372</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>2019</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6/13/2021 13:25:49</td>\n",
       "      <td>6/13/2021 13:27:28</td>\n",
       "      <td>213.127.82.232</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>UgwWKCWtSJdFvjGHvTp4AaABAg.8kUC5dGrQ2H8kUDRihE2f3</td>\n",
       "      <td>1206</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>2018</td>\n",
       "      <td>2018.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>11/30/2021 17:24:18</td>\n",
       "      <td>11/30/2021 17:27:16</td>\n",
       "      <td>213.127.109.191</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>Ugw2eTvkZLfH9MDVg1R4AaABAg</td>\n",
       "      <td>1214</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>662.0</td>\n",
       "      <td>9472.0</td>\n",
       "      <td>743.0</td>\n",
       "      <td>2912.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>economy</td>\n",
       "      <td>3630000.0</td>\n",
       "      <td>1</td>\n",
       "      <td>2019</td>\n",
       "      <td>2018.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6/18/2021 17:59:54</td>\n",
       "      <td>6/18/2021 18:01:59</td>\n",
       "      <td>213.127.126.118</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>Ugi7W7rSRtQSQngCoAEC.8SSDZOcrB8y8SSIYSKC0FI</td>\n",
       "      <td>1963</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>2017</td>\n",
       "      <td>2017.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 81 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             StartDate         RecordedDate        IPAddress  Finished  Coder  \\\n",
       "0   5/30/2021 13:03:17   5/30/2021 13:04:17     62.194.51.29         1      6   \n",
       "1    9/9/2021 18:49:48    9/9/2021 18:51:32    213.127.110.0         1      6   \n",
       "2   6/13/2021 13:25:49   6/13/2021 13:27:28   213.127.82.232         1      6   \n",
       "3  11/30/2021 17:24:18  11/30/2021 17:27:16  213.127.109.191         1      6   \n",
       "4   6/18/2021 17:59:54   6/18/2021 18:01:59  213.127.126.118         1      6   \n",
       "\n",
       "                                                  ID  Mark_ID  Genre  \\\n",
       "0  UgyPHwv8G0cDE6-wEgl4AaABAg.8_0ZjJKSJty8_0kXGkAd2U      119      0   \n",
       "1                                1110578710648890000      372      2   \n",
       "2  UgwWKCWtSJdFvjGHvTp4AaABAg.8kUC5dGrQ2H8kUDRihE2f3     1206      0   \n",
       "3                         Ugw2eTvkZLfH9MDVg1R4AaABAg     1214      1   \n",
       "4        Ugi7W7rSRtQSQngCoAEC.8SSDZOcrB8y8SSIYSKC0FI     1963      0   \n",
       "\n",
       "   topiccode  Platform  ...  dislikeCount_video likeCount_video  \\\n",
       "0          0         1  ...                 NaN             NaN   \n",
       "1          4         2  ...                 NaN             NaN   \n",
       "2          0         1  ...                 NaN             NaN   \n",
       "3          2         1  ...               662.0          9472.0   \n",
       "4          0         1  ...                 NaN             NaN   \n",
       "\n",
       "   date_difference  commentCount_video  replyCount_comment    topic  \\\n",
       "0              NaN                 NaN                 NaN      NaN   \n",
       "1              NaN                 NaN                 NaN      NaN   \n",
       "2              NaN                 NaN                 NaN      NaN   \n",
       "3            743.0              2912.0                 0.0  economy   \n",
       "4              NaN                 NaN                 NaN      NaN   \n",
       "\n",
       "   subscribers  HATELIST_FOCUSED_DUMMY  Time_comment_year Time_video_year  \n",
       "0          NaN                       0               2017          2017.0  \n",
       "1          NaN                       0               2019             NaN  \n",
       "2          NaN                       0               2018          2018.0  \n",
       "3    3630000.0                       1               2019          2018.0  \n",
       "4          NaN                       1               2017          2017.0  \n",
       "\n",
       "[5 rows x 81 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainset.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "a9aec51e",
   "metadata": {},
   "outputs": [],
   "source": [
    "#add train/test split variable to the dataset:\n",
    "dataset.loc[dataset['Mark_ID'].isin(trainset['Mark_ID']), 'split'] = 'train'\n",
    "dataset.loc[dataset['Mark_ID'].isin(testset['Mark_ID']), 'split'] = 'test'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "ba49bcf4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "split\n",
       "train    3089\n",
       "test      773\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset['split'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "e043564f",
   "metadata": {},
   "outputs": [],
   "source": [
    "#set option variables:\n",
    "\n",
    "#set options to low temperature (0,1):\n",
    "options_low_str = \"\"\"\n",
    "seed: 42\n",
    "temperature: 0.1\n",
    "\"\"\"\n",
    "\n",
    "options_low = yaml.safe_load(options_low_str)\n",
    "\n",
    "MODELsmall: str = 'llama3.1:8b-instruct-q6_K' # options: 'gemma:7b-instruct-q6_K', 'gemma2:27b-instruct-q6_K', 'llama3.1:8b-instruct-q6_K', 'llama3.1:70b-instruct-q6_K', 'mistral:7b-instruct-v0.3-q6_K', 'mistral-large:123b-instruct-2407-q6_K', 'mixtral:8x7b-instruct-v0.1-q6_K', 'mixtral:8x22b-instruct-v0.1-q6_K', 'phi3:14b-medium-128k-instruct-q6_K' or 'qwen2:72b-instruct-q6_K'\n",
    "MODELlarge: str = 'llama3.1:70b-instruct-q6_K' # options: 'gemma:7b-instruct-q6_K', 'gemma2:27b-instruct-q6_K', 'llama3.1:8b-instruct-q6_K', 'llama3.1:70b-instruct-q6_K', 'mistral:7b-instruct-v0.3-q6_K', 'mistral-large:123b-instruct-2407-q6_K', 'mixtral:8x7b-instruct-v0.1-q6_K', 'mixtral:8x22b-instruct-v0.1-q6_K', 'phi3:14b-medium-128k-instruct-q6_K' or 'qwen2:72b-instruct-q6_K'\n",
    "MODELgpt4o = \"nf-gpt-4o-2024-08-06\" # in principe is er nu van elk model een nf (no filter) en een normale versie beschikbaar, de no filter versies zijn alleen voor onderzoekers beschikbaar voor analyze van content die niet door de filter heen zou komen.\n",
    "MODELgpt4T = \"nf-gpt-4-turbo\" # Can be gpt-35-turbo, gpt-4-turbo, gpt-4 or Meta-Llama-3-8B-Instruct.\n",
    "MODELgpt4 = \"nf-gpt-4\" # Can be gpt-35-turbo, gpt-4-turbo, gpt-4 or Meta-Llama-3-8B-Instruct.\n",
    "\n",
    "options_zero_str = \"\"\"\n",
    "seed: 42\n",
    "temperature: 0\n",
    "\"\"\"\n",
    "options_zero = yaml.safe_load(options_zero_str)\n",
    "\n",
    "temperature_0 : int = 0\n",
    "SEED: int = 42\n",
    "MAX10: int = 10\n",
    "TOPP1: int = 1\n",
    "\n",
    "\n",
    "options_large_str = \"\"\"\n",
    "seed: 42\n",
    "temperature: 0\n",
    "num_predict: 2000\n",
    "\"\"\"\n",
    "options_large = yaml.safe_load(options_large_str)\n",
    "\n",
    "#load environment variables:\n",
    "api_key = os.environ.get('sjoerd_key')\n",
    "\n",
    "#setttings:\n",
    "api_endpoint = \"https://ai-research-proxy.azurewebsites.net/chat/completions\"\n",
    "api_endpoint_comp = \"https://ai-research-proxy.azurewebsites.net/completions\"\n",
    "####### API REQUEST FORMATTING ######\n",
    "headers = {\n",
    "    \"Content-Type\": \"application/json\",\n",
    "    \"Authorization\": \"Bearer \" + api_key\n",
    "}\n",
    "\n",
    "def load_json(path: str):\n",
    "    with open(path, encoding='utf-8') as fp:\n",
    "        return json.load(fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e3e2e8ad",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "5it [00:24,  4.94s/it]\n"
     ]
    }
   ],
   "source": [
    "#inspect GLLM reasoning to ensure valid processing of task and correct task interpretation:\n",
    "            \n",
    "predictions: typing.Dict[str, np.ndarray] = {}\n",
    "for index, row in tqdm.tqdm(dataset[\"commentText\"][:5].items()):\n",
    "    try: \n",
    "        output = np.array(\n",
    "            requests.post(\n",
    "                'https://inf.cl.uni-trier.de/',\n",
    "                json={\n",
    "                    'model': MODELsmall,\n",
    "                    'prompt': f\"Does this comment acknowledge a previously posted user-comment or claim of another discussant? \\nInstruction: By referring back to another comment, one recognizes the existence of a previous comment. Acknowledgment can be neutral or by explicitly endorsing another discussant's comment (e.g. 'I absolutely agree with what you're saying'). Keywords such as 'Yes; Yep; Exactly; I totally agree' may signal acknowledgement of a comment of another discussant. Acknowledgement can also be explicitly disagreeing with another discussant's comment or claim (e.g. '@RoniBox No, you're not right. Climate change is not real!'). \\nA comment coded as YES for Acknowledgement is also likely to be coded as YES for Interaction. \\nExample 1: '@tirtha simanta yeah but now someone deleted all of it' \\nExample 2: 'Nope, that is not true' (is 1 under isReply) \\nException 1: Code as NO if there is NOT a @username (or +1username, or username) NOR it is NOT a reply, because in most cases that means the comment is not referring to a commenter involved in the discussion. Whether this comment is a reply or not is indicated by 'isReply' ('1' means it is a reply) \\nException 2: Code as NO if the comment is simply criticizing or cursing at another discussant (e.g. @Roompie Think properly, you fool), without explicitly saying that the writer (dis)agrees with another discussant. \\\\n\\\\n First explain your reasoning, then respond with the predicted class (0 or 1) of the request.\\\\n\\\\nText: {row}\"                        \n",
    "                    }).json()['response'])                  \n",
    "        \n",
    "                    \n",
    "    except json.JSONDecodeError:\n",
    "        print(\"invalid json response, skipping to next batch\")\n",
    "        output = None\n",
    "        \n",
    "    predictions[index] = output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "c5272c27",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0: array(' This comment does not acknowledge a previously posted user-comment or claim of another discussant. The text simply says \"sad\" which doesn\\'t refer to any specific comment or user. Therefore, I would predict that this comment is coded as NO for Acknowledgement and also NO for Interaction since it doesn\\'t interact with any other comments.',\n",
       "       dtype='<U338'),\n",
       " 1: array(' This comment does not appear to acknowledge a previously posted user-comment or claim of another discussant, as there is no explicit reference to or endorsement of another user\\'s comment. The comment contains a playful tone, addressing \"a box of rocks\" and claiming to represent the \"coalition for mineral rights,\" but it does not engage with any specific points raised by other users. Therefore, I would predict that this request should be coded as 0 (NO) for Acknowledgement.\\n\\nThat being said, without more context, it is possible that this comment is a reply to a specific user or thread of comments that are not included in the provided text. If that is the case, my answer may change based on the content and tone of those previous comments.',\n",
       "       dtype='<U747'),\n",
       " 2: array(' This comment does not appear to acknowledge a previously posted user-comment or claim of another discussant. Although the comment starts with \"@colbertlateshow,\" it is followed by a link and an incomplete sentence, which suggests that it may not be directly responding to a previous comment but rather addressing the broader topic of compromise involving someone named \"trump tow\" (likely referring to former President Donald Trump). Therefore, I would predict a class of 0 for this request.',\n",
       "       dtype='<U492'),\n",
       " 3: array(' This comment does acknowledge a previously posted user-comment or claim of another discussant, and I will explain my reasoning below.\\n\\nThe text \"Try reading the comments on both Fox News and The Daily Show clips\" suggests that the writer is responding to a conversation about the comments section on those two platforms (Fox News and The Daily Show). By referring to these specific platforms, the writer acknowledges the existence of a previous comment or discussion about them. Therefore, I would predict this text to be coded as YES for Acknowledgement.\\n\\nAdditionally, the tone of this comment seems to be in agreement with the previous comment or discussion, implying that there was some shared concern or criticism about the state of free and civil dialogue on these platforms. However, without more context or information about the preceding conversation, it is difficult to determine whether this text would also be coded as YES for Interaction.\\n\\nOverall, based on the available information, I would predict this comment to be coded as YES for Acknowledgement.',\n",
       "       dtype='<U1067'),\n",
       " 4: array(' The text \"No-one else will hug him.\" does not acknowledge a previously posted user-comment or claim of another discussant. This is because there is no reference to any other comment or user in the text, nor is it a reply to any previous comment. I would predict this as class 0 (NO).',\n",
       "       dtype='<U284')}"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictions\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f291cecb",
   "metadata": {},
   "source": [
    "apparently, it Llama3.1 expects the input text to mention the isReply-variable (which it does not), which might bias the results of the coding\n",
    "mixtral however seems to ignore this part of the instruction\n",
    "overall the reasoning provided does not indicate serious interpretation problems of the model with regards to the instructions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "9420b18f",
   "metadata": {},
   "outputs": [],
   "source": [
    "#make a list of prompts to be classified:\n",
    "pubsphereprompts = ['diversity_disagreement', 'diversity_ideological_direction', 'diversity_positiondum', 'incivility_allcaps', 'incivility_attackreputation', 'incivility_namecalling', 'incivility_question_intelligence', 'incivility_sarcasm', 'incivility_vulgarity', 'interactivity_acknowledgement', 'intolerance_discrimination', 'intolerance_rights', 'intolerance_violence', 'political_ideology_US', 'rationality_background_info', 'rationality_external_evidence', 'rationality_reasoning', 'rationality_topic_relevance']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "49c54216",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "classifying incivility_namecalling: 100%|██████████| 3862/3862 [29:41<00:00,  2.17it/s]   \n",
      "classifying incivility_namecalling_edit: 100%|██████████| 3862/3862 [28:54<00:00,  2.23it/s]   \n"
     ]
    }
   ],
   "source": [
    "pubspherepromptsrun14 = ['incivility_namecalling', 'incivility_namecalling_edit']   \n",
    "predictions14: typing.Dict[str, np.ndarray] = {\n",
    "    label: (\n",
    "        src.PromptClassify\n",
    "        .from_json(path)\n",
    "        (dataset[\"commentText\"], model=MODELlarge, options=options_low)\n",
    "    )\n",
    "    for label, path in CFG.prompt_classify_files.items() if label in pubspherepromptsrun14\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "46d0017e",
   "metadata": {},
   "outputs": [],
   "source": [
    "#load the dataset:\n",
    "dataset_w_pred_2 = pd.read_json(f'{CFG.report_dir}/publicsphere.cardiff_prompt_classify_s.json')\n",
    "dataset_w_pred_anon = pd.read_parquet('data/publicsphere/publicsphere.cardiff_prompt_classify_anon.parquet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c2bf2ccc",
   "metadata": {},
   "outputs": [],
   "source": [
    "#add train/test split variable to the dataset:\n",
    "dataset_w_pred_anon.loc[dataset_w_pred_anon['Mark_ID'].isin(trainset['Mark_ID']), 'split'] = 'train'\n",
    "dataset_w_pred_anon.loc[dataset_w_pred_anon['Mark_ID'].isin(testset['Mark_ID']), 'split'] = 'test'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "a0916464",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "incivility_namecalling\n",
      "No     2670\n",
      "Yes    1189\n",
      "Name: count, dtype: int64\n",
      "------------------------------------------\n",
      "incivility_namecalling_edit\n",
      "No     2486\n",
      "Yes    1371\n",
      "Name: count, dtype: int64\n",
      "------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "#join to the dataset:   \n",
    "for _, preds in predictions14.items():\n",
    "    print(preds.value_counts())\n",
    "    print(\"-\" * 42)\n",
    "    dataset_w_pred_2 = dataset_w_pred_2.join(preds, rsuffix='_updated')\n",
    "    dataset_w_pred_anon = dataset_w_pred_anon.join(preds, rsuffix='_updated')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "145f7026",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['StartDate',\n",
       " 'RecordedDate',\n",
       " 'IPAddress',\n",
       " 'Finished',\n",
       " 'Coder',\n",
       " 'ID',\n",
       " 'Mark_ID',\n",
       " 'Genre',\n",
       " 'topiccode',\n",
       " 'Platform',\n",
       " 'Anonymity',\n",
       " 'Anonymity_9_TEXT',\n",
       " 'codable',\n",
       " 'Interaction',\n",
       " 'Acknowledgement',\n",
       " 'TopicRelevance',\n",
       " 'Reasoning',\n",
       " 'BackgroundInfo',\n",
       " 'ExternalEvidence',\n",
       " 'ExternalEvidence_1_TEXT',\n",
       " 'Opinion',\n",
       " 'disagreement',\n",
       " 'Ideologicaldirection',\n",
       " 'Name_calling',\n",
       " 'Vulgarity',\n",
       " 'Attack_reputation',\n",
       " 'Question_Intelligenc',\n",
       " 'All_caps_function',\n",
       " 'Sarcasm_to_criticize',\n",
       " 'Individual_right',\n",
       " 'discrimination',\n",
       " 'Invoke_violence',\n",
       " 'Tone',\n",
       " 'INTERACTIVITY_DUMMY',\n",
       " 'RATIONALITY_DUMMY',\n",
       " 'HAS_OPINION_DUMMY',\n",
       " 'LIBERAL_NEUTRAL_CONSERVATIVE',\n",
       " 'LIBERAL_DUMMY',\n",
       " 'CONSERVATIVE_DUMMY',\n",
       " 'NAMECALLING_DUMMY',\n",
       " 'VULGAR_DUMMY',\n",
       " 'NAMECALLING_VULGAR_DUMMY',\n",
       " 'INCIVILITY_ORDINAL',\n",
       " 'INCIVILITY_DUMMY',\n",
       " 'INTOLERANCE_DUMMY',\n",
       " 'filter_$',\n",
       " 'IMPOLITENESS_DUMMY',\n",
       " 'showName',\n",
       " 'genre',\n",
       " 'Time_comment',\n",
       " 'likeCount_comment',\n",
       " 'entities',\n",
       " 'place',\n",
       " 'retweet_count',\n",
       " 'platform',\n",
       " 'retweeted',\n",
       " 'language',\n",
       " 'source',\n",
       " 'in_reply_to_status_id_str',\n",
       " 'in_reply_to_user_id_str',\n",
       " 'in_reply_to_screen_name',\n",
       " 'is_quote_status',\n",
       " 'videoTitle',\n",
       " 'description',\n",
       " 'Time_video',\n",
       " 'channelTitle',\n",
       " 'channelId',\n",
       " 'viewCount',\n",
       " 'dislikeCount_video',\n",
       " 'likeCount_video',\n",
       " 'date_difference',\n",
       " 'commentCount_video',\n",
       " 'replyCount_comment',\n",
       " 'topic',\n",
       " 'subscribers',\n",
       " 'HATELIST_FOCUSED_DUMMY',\n",
       " 'Time_comment_year',\n",
       " 'Time_video_year',\n",
       " 'interactivity_acknowledgement',\n",
       " 'rationality_external_evidence',\n",
       " 'rationality_topic_relevance',\n",
       " 'political_negativity',\n",
       " 'rationality_background_info',\n",
       " 'rationality_reasoning',\n",
       " 'topics',\n",
       " 'emotions',\n",
       " 'sentiment',\n",
       " 'irony',\n",
       " 'offensive',\n",
       " 'hate',\n",
       " 'commentText',\n",
       " 'political_ideology_US',\n",
       " 'diversity_disagreement',\n",
       " 'diversity_ideologicaldirection',\n",
       " 'diversity_position',\n",
       " 'incivility_allcaps',\n",
       " 'incivility_attackreputation',\n",
       " 'incivility_namecalling',\n",
       " 'incivility_question_intelligence',\n",
       " 'incivility_sarcasm',\n",
       " 'incivility_vulgarity',\n",
       " 'diversity_disagreement_dum',\n",
       " 'diversity_position_dum',\n",
       " 'incivility_allcaps_dum',\n",
       " 'incivility_attackreputation_dum',\n",
       " 'incivility_question_intelligence_dum',\n",
       " 'incivility_sarcasm_dum',\n",
       " 'incivility_namecalling_dum',\n",
       " 'incivility_vulgarity_dum',\n",
       " 'diversity_ideologicaldirection_dum',\n",
       " 'interactivity_acknowledgement_low',\n",
       " 'intolerance_discrimination',\n",
       " 'intolerance_rights',\n",
       " 'intolerance_violence',\n",
       " 'political_ideology_US_low',\n",
       " 'rationality_background_info_low',\n",
       " 'rationality_external_evidence_low',\n",
       " 'rationality_reasoning_low',\n",
       " 'rationality_topic_relevance_low',\n",
       " 'incivility_namecalling_low2',\n",
       " 'rationality_background_info_updated',\n",
       " 'rationality_external_evidence_updated',\n",
       " 'rationality_background_info_run2',\n",
       " 'rationality_external_evidence_run2',\n",
       " 'rationality_external_evidence_edit',\n",
       " 'rationality_background_info_edit',\n",
       " 'interactivity_acknowledgement_updated',\n",
       " 'rationality_reasoning_edit',\n",
       " 'incivility_namecalling_updated',\n",
       " 'incivility_namecalling_edit']"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset_w_pred_2.columns.to_list()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "9664252f",
   "metadata": {},
   "outputs": [],
   "source": [
    "#need to recode all the llama generated variables to 0,1:\n",
    "dataset_w_pred_anon.loc[:, 'diversity_positiondum_dum'] = dataset_w_pred_anon.loc[:, 'diversity_positiondum'].map({\"Yes\": 1, \"No\":0}).fillna(0).astype(int)\n",
    "dataset_w_pred_anon.loc[:, 'diversity_disagreement_dum'] = dataset_w_pred_anon.loc[:, 'diversity_disagreement'].map({\"Yes\": 1, \"No\":0}).fillna(0).astype(int)\n",
    "dataset_w_pred_anon.loc[:, 'diversity_position_dum'] = dataset_w_pred_anon.loc[:, 'diversity_position'].map({\"Yes\": 1, \"No\":0}).fillna(0).astype(int)\n",
    "dataset_w_pred_anon.loc[:, 'incivility_allcaps_dum'] = dataset_w_pred_anon.loc[:, 'incivility_allcaps'].map({\"Yes\": 1, \"No\":0}).fillna(0).astype(int)\n",
    "dataset_w_pred_anon.loc[:, 'incivility_attackreputation_dum'] = dataset_w_pred_anon.loc[:, 'incivility_attackreputation'].map({\"Yes\": 1, \"No\":0}).fillna(0).astype(int)\n",
    "dataset_w_pred_anon.loc[:, 'incivility_question_intelligence_dum'] = dataset_w_pred_anon.loc[:, 'incivility_question_intelligence'].map({\"Yes\": 1, \"No\":0}).fillna(0).astype(int)\n",
    "dataset_w_pred_anon.loc[:, 'incivility_sarcasm_dum'] = dataset_w_pred_anon.loc[:, 'incivility_sarcasm'].map({\"Yes\": 1, \"No\":0}).fillna(0).astype(int)\n",
    "dataset_w_pred_anon.loc[:, 'incivility_namecalling_dum'] = dataset_w_pred_anon.loc[:, 'incivility_namecalling'].map({\"Yes\": 1, \"No\":0}).fillna(0).astype(int)\n",
    "dataset_w_pred_anon.loc[:, 'incivility_vulgarity_dum'] = dataset_w_pred_anon.loc[:, 'incivility_vulgarity'].map({\"Yes\": 1, \"No\":0}).fillna(0).astype(int)\n",
    "\n",
    "dataset_w_pred_anon.loc[:, 'diversity_ideologicaldirection_dum'] = dataset_w_pred_anon.loc[:, 'diversity_ideologicaldirection'].map({\"Left/Liberal/Democratic\": 0, \"Absent\":1, \"Neutral\":1, \"Unclear which direction\":1 , \"Right/Conservative/Republican\":2}).fillna(0).astype(int)\n",
    "\n",
    "dataset_w_pred_2.loc[:, 'diversity_disagreement_dum'] = dataset_w_pred_2.loc[:, 'diversity_disagreement'].map({\"Yes\": 1, \"No\":0}).fillna(0).astype(int)\n",
    "dataset_w_pred_2.loc[:, 'diversity_position_dum'] = dataset_w_pred_2.loc[:, 'diversity_position'].map({\"Yes\": 1, \"No\":0}).fillna(0).astype(int)\n",
    "dataset_w_pred_2.loc[:, 'incivility_allcaps_dum'] = dataset_w_pred_2.loc[:, 'incivility_allcaps'].map({\"Yes\": 1, \"No\":0}).fillna(0).astype(int)\n",
    "dataset_w_pred_2.loc[:, 'incivility_attackreputation_dum'] = dataset_w_pred_2.loc[:, 'incivility_attackreputation'].map({\"Yes\": 1, \"No\":0}).fillna(0).astype(int)\n",
    "dataset_w_pred_2.loc[:, 'incivility_question_intelligence_dum'] = dataset_w_pred_2.loc[:, 'incivility_question_intelligence'].map({\"Yes\": 1, \"No\":0}).fillna(0).astype(int)\n",
    "dataset_w_pred_2.loc[:, 'incivility_sarcasm_dum'] = dataset_w_pred_2.loc[:, 'incivility_sarcasm'].map({\"Yes\": 1, \"No\":0}).fillna(0).astype(int)\n",
    "dataset_w_pred_2.loc[:, 'incivility_namecalling_dum'] = dataset_w_pred_2.loc[:, 'incivility_namecalling'].map({\"Yes\": 1, \"No\":0}).fillna(0).astype(int)\n",
    "dataset_w_pred_2.loc[:, 'incivility_vulgarity_dum'] = dataset_w_pred_2.loc[:, 'incivility_vulgarity'].map({\"Yes\": 1, \"No\":0}).fillna(0).astype(int)\n",
    "\n",
    "dataset_w_pred_2.loc[:, 'diversity_ideologicaldirection_dum'] = dataset_w_pred_2.loc[:, 'diversity_ideologicaldirection'].map({\"Left/Liberal/Democratic\": 0, \"Absent\":1, \"Neutral\":1, \"Unclear which direction\":1 , \"Right/Conservative/Republican\":2}).fillna(0).astype(int)\n",
    "\n",
    "#need to combine incivility and rationality items into seperate variables:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "5d19e071",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>StartDate</th>\n",
       "      <th>RecordedDate</th>\n",
       "      <th>IPAddress</th>\n",
       "      <th>Finished</th>\n",
       "      <th>Coder</th>\n",
       "      <th>ID</th>\n",
       "      <th>Mark_ID</th>\n",
       "      <th>Genre</th>\n",
       "      <th>topiccode</th>\n",
       "      <th>Platform</th>\n",
       "      <th>...</th>\n",
       "      <th>incivility_prompt_dum</th>\n",
       "      <th>hate_list_prompt_dum</th>\n",
       "      <th>rationality_background_info</th>\n",
       "      <th>rationality_external_evidence</th>\n",
       "      <th>rationality_external_evidence_edit</th>\n",
       "      <th>rationality_background_info_edit</th>\n",
       "      <th>interactivity_acknowledgement</th>\n",
       "      <th>rationality_reasoning_edit</th>\n",
       "      <th>incivility_namecalling_updated</th>\n",
       "      <th>incivility_namecalling_edit</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5/30/2021 13:03:17</td>\n",
       "      <td>5/30/2021 13:04:17</td>\n",
       "      <td>62.194.51.29</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>UgyPHwv8G0cDE6-wEgl4AaABAg.8_0ZjJKSJty8_0kXGkAd2U</td>\n",
       "      <td>119</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>10/11/2021 10:34:05</td>\n",
       "      <td>10/11/2021 10:36:46</td>\n",
       "      <td>213.127.109.191</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>Ugx2WXq9UdV8mPPjejJ4AaABAg.8yHCKV0Boe58yYRxEQEF45</td>\n",
       "      <td>282</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>9/9/2021 18:49:48</td>\n",
       "      <td>9/9/2021 18:51:32</td>\n",
       "      <td>213.127.110.0</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>1110578710648890000</td>\n",
       "      <td>372</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6/6/2021 16:12:46</td>\n",
       "      <td>6/6/2021 16:16:16</td>\n",
       "      <td>213.127.76.145</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>UgwUPFScjJ0MCeaP2F54AaABAg.8lvp3fc9Euf8lvvgsUgEgV</td>\n",
       "      <td>769</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6/13/2021 13:25:49</td>\n",
       "      <td>6/13/2021 13:27:28</td>\n",
       "      <td>213.127.82.232</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>UgwWKCWtSJdFvjGHvTp4AaABAg.8kUC5dGrQ2H8kUDRihE2f3</td>\n",
       "      <td>1206</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 160 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             StartDate         RecordedDate        IPAddress  Finished  Coder  \\\n",
       "0   5/30/2021 13:03:17   5/30/2021 13:04:17     62.194.51.29         1      6   \n",
       "1  10/11/2021 10:34:05  10/11/2021 10:36:46  213.127.109.191         1      6   \n",
       "2    9/9/2021 18:49:48    9/9/2021 18:51:32    213.127.110.0         1      6   \n",
       "3    6/6/2021 16:12:46    6/6/2021 16:16:16   213.127.76.145         1      6   \n",
       "4   6/13/2021 13:25:49   6/13/2021 13:27:28   213.127.82.232         1      6   \n",
       "\n",
       "                                                  ID  Mark_ID  Genre  \\\n",
       "0  UgyPHwv8G0cDE6-wEgl4AaABAg.8_0ZjJKSJty8_0kXGkAd2U      119      0   \n",
       "1  Ugx2WXq9UdV8mPPjejJ4AaABAg.8yHCKV0Boe58yYRxEQEF45      282      1   \n",
       "2                                1110578710648890000      372      2   \n",
       "3  UgwUPFScjJ0MCeaP2F54AaABAg.8lvp3fc9Euf8lvvgsUgEgV      769      0   \n",
       "4  UgwWKCWtSJdFvjGHvTp4AaABAg.8kUC5dGrQ2H8kUDRihE2f3     1206      0   \n",
       "\n",
       "   topiccode  Platform  ...  incivility_prompt_dum hate_list_prompt_dum  \\\n",
       "0          0         1  ...                      0                    0   \n",
       "1          2         1  ...                      1                    1   \n",
       "2          4         2  ...                      1                    0   \n",
       "3          0         1  ...                      0                    0   \n",
       "4          0         1  ...                      1                    0   \n",
       "\n",
       "   rationality_background_info  rationality_external_evidence  \\\n",
       "0                           No                             No   \n",
       "1                           No                             No   \n",
       "2                          Yes                            Yes   \n",
       "3                           No                             No   \n",
       "4                           No                             No   \n",
       "\n",
       "   rationality_external_evidence_edit  rationality_background_info_edit  \\\n",
       "0                                  No                                No   \n",
       "1                                  No                                No   \n",
       "2                                 Yes                               Yes   \n",
       "3                                 Yes                               Yes   \n",
       "4                                  No                                No   \n",
       "\n",
       "   interactivity_acknowledgement  rationality_reasoning_edit  \\\n",
       "0                             No                          No   \n",
       "1                             No                         Yes   \n",
       "2                            Yes                         Yes   \n",
       "3                             No                         Yes   \n",
       "4                             No                          No   \n",
       "\n",
       "   incivility_namecalling_updated incivility_namecalling_edit  \n",
       "0                              No                          No  \n",
       "1                             Yes                         Yes  \n",
       "2                              No                          No  \n",
       "3                              No                          No  \n",
       "4                              No                          No  \n",
       "\n",
       "[5 rows x 160 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset_w_pred_anon.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "id": "ccdf1bb2",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_w_pred_2.to_json(f'{CFG.report_dir}/publicsphere.cardiff_prompt_classify_s.json', orient=\"records\", force_ascii=False, indent=4)\n",
    "dataset_w_pred_2.to_parquet(f'{CFG.report_dir}/publicsphere.cardiff_prompt_classify_s.parquet')\n",
    "dataset_w_pred_anon.to_json('data/publicsphere/publicsphere.cardiff_prompt_classify_anon.json', orient=\"records\", force_ascii=False, indent=4)\n",
    "dataset_w_pred_anon.to_parquet('data/publicsphere/publicsphere.cardiff_prompt_classify_anon.parquet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "005b07d0",
   "metadata": {},
   "outputs": [],
   "source": [
    "#load data\n",
    "dataset_w_pred_anon = pd.read_parquet('data/publicsphere/publicsphere.cardiff_prompt_classify_anon.parquet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "f02004b4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['StartDate',\n",
       " 'RecordedDate',\n",
       " 'IPAddress',\n",
       " 'Finished',\n",
       " 'Coder',\n",
       " 'ID',\n",
       " 'Mark_ID',\n",
       " 'Genre',\n",
       " 'topiccode',\n",
       " 'Platform',\n",
       " 'Anonymity',\n",
       " 'Anonymity_9_TEXT',\n",
       " 'codable',\n",
       " 'Interaction',\n",
       " 'Acknowledgement',\n",
       " 'TopicRelevance',\n",
       " 'Reasoning',\n",
       " 'BackgroundInfo',\n",
       " 'ExternalEvidence',\n",
       " 'ExternalEvidence_1_TEXT',\n",
       " 'Opinion',\n",
       " 'disagreement',\n",
       " 'Ideologicaldirection',\n",
       " 'Name_calling',\n",
       " 'Vulgarity',\n",
       " 'Attack_reputation',\n",
       " 'Question_Intelligenc',\n",
       " 'All_caps_function',\n",
       " 'Sarcasm_to_criticize',\n",
       " 'Individual_right',\n",
       " 'discrimination',\n",
       " 'Invoke_violence',\n",
       " 'Tone',\n",
       " 'INTERACTIVITY_DUMMY',\n",
       " 'RATIONALITY_DUMMY',\n",
       " 'HAS_OPINION_DUMMY',\n",
       " 'LIBERAL_NEUTRAL_CONSERVATIVE',\n",
       " 'LIBERAL_DUMMY',\n",
       " 'CONSERVATIVE_DUMMY',\n",
       " 'NAMECALLING_DUMMY',\n",
       " 'VULGAR_DUMMY',\n",
       " 'NAMECALLING_VULGAR_DUMMY',\n",
       " 'INCIVILITY_ORDINAL',\n",
       " 'INCIVILITY_DUMMY',\n",
       " 'INTOLERANCE_DUMMY',\n",
       " 'filter_$',\n",
       " 'IMPOLITENESS_DUMMY',\n",
       " 'showName',\n",
       " 'genre',\n",
       " 'Time_comment',\n",
       " 'likeCount_comment',\n",
       " 'entities',\n",
       " 'place',\n",
       " 'retweet_count',\n",
       " 'platform',\n",
       " 'retweeted',\n",
       " 'language',\n",
       " 'source',\n",
       " 'in_reply_to_status_id_str',\n",
       " 'in_reply_to_user_id_str',\n",
       " 'in_reply_to_screen_name',\n",
       " 'is_quote_status',\n",
       " 'videoTitle',\n",
       " 'description',\n",
       " 'Time_video',\n",
       " 'channelTitle',\n",
       " 'channelId',\n",
       " 'viewCount',\n",
       " 'dislikeCount_video',\n",
       " 'likeCount_video',\n",
       " 'date_difference',\n",
       " 'commentCount_video',\n",
       " 'replyCount_comment',\n",
       " 'topic',\n",
       " 'subscribers',\n",
       " 'HATELIST_FOCUSED_DUMMY',\n",
       " 'Time_comment_year',\n",
       " 'Time_video_year',\n",
       " 'interactivity_acknowledgement_old',\n",
       " 'rationality_external_evidence_old',\n",
       " 'rationality_topic_relevance_old',\n",
       " 'political_negativity_old',\n",
       " 'rationality_background_info_old',\n",
       " 'rationality_reasoning_old',\n",
       " 'topics',\n",
       " 'emotions',\n",
       " 'sentiment',\n",
       " 'irony',\n",
       " 'offensive',\n",
       " 'hate',\n",
       " 'political_ideology_US_old',\n",
       " 'offensive_dum',\n",
       " 'hate_dum',\n",
       " 'cardiff_incivil',\n",
       " 'rationality_reasoning_dum_old',\n",
       " 'rationality_background_info_dum_old',\n",
       " 'rationality_external_evidence_dum_old',\n",
       " 'rationality_prompt_dum_old',\n",
       " 'political_conservative_US_old',\n",
       " 'political_liberal_US_old',\n",
       " 'political_opinion_US_old',\n",
       " 'diversity_positiondum_old',\n",
       " 'diversity_disagreement',\n",
       " 'diversity_ideologicaldirection',\n",
       " 'diversity_position',\n",
       " 'incivility_allcaps',\n",
       " 'incivility_attackreputation',\n",
       " 'incivility_question_intelligence',\n",
       " 'incivility_sarcasm',\n",
       " 'incivility_namecalling_NA',\n",
       " 'incivility_vulgarity',\n",
       " 'diversity_positiondum_dum_old',\n",
       " 'diversity_disagreement_dum',\n",
       " 'diversity_position_dum',\n",
       " 'incivility_allcaps_dum',\n",
       " 'incivility_attackreputation_dum',\n",
       " 'incivility_question_intelligence_dum',\n",
       " 'incivility_sarcasm_dum',\n",
       " 'incivility_namecalling_dum_NA',\n",
       " 'incivility_vulgarity_dum',\n",
       " 'diversity_ideologicaldirection_dum',\n",
       " 'interactivity_acknowledgement_low',\n",
       " 'intolerance_discrimination',\n",
       " 'intolerance_rights',\n",
       " 'intolerance_violence',\n",
       " 'political_ideology_US_low',\n",
       " 'rationality_background_info_low',\n",
       " 'rationality_external_evidence_low',\n",
       " 'rationality_reasoning_low',\n",
       " 'rationality_topic_relevance_low',\n",
       " 'incivility_namecalling',\n",
       " 'rationality_background_info_updated',\n",
       " 'rationality_external_evidence_updated',\n",
       " 'interactivity_acknowledgement_dum',\n",
       " 'rationality_external_evidence_dum',\n",
       " 'rationality_topic_relevance_dum',\n",
       " 'rationality_background_info_low_dum',\n",
       " 'rationality_reasoning_dum',\n",
       " 'intolerance_discrimination_dum',\n",
       " 'intolerance_rights_dum',\n",
       " 'intolerance_violence_dum',\n",
       " 'incivility_namecalling_dum',\n",
       " 'political_conservative_US_dum',\n",
       " 'political_liberal_US_dum',\n",
       " 'political_opinion_US_dum',\n",
       " 'diversity_conservative__dum',\n",
       " 'diversity_liberal_dum',\n",
       " 'rationality_background_info_updated_dum',\n",
       " 'rationality_external_evidence_updated_dum',\n",
       " 'rationality_prompt_dum',\n",
       " 'incivility_prompt_dum',\n",
       " 'hate_list_prompt_dum',\n",
       " 'rationality_background_info_run2',\n",
       " 'rationality_external_evidence_run2',\n",
       " 'rationality_external_evidence_edit',\n",
       " 'rationality_background_info_edit',\n",
       " 'interactivity_acknowledgement_updated',\n",
       " 'rationality_reasoning_edit',\n",
       " 'incivility_namecalling_updated',\n",
       " 'incivility_namecalling_edit',\n",
       " 'diversity_conservative_dum',\n",
       " 'interactivity_acknowledgement_updated_dum',\n",
       " 'incivility_namecalling_updated_dum',\n",
       " 'incivility_namecalling_edit_dum',\n",
       " 'rationality_reasoning_edit_dum',\n",
       " 'rationality_background_info_run2_dum',\n",
       " 'rationality_external_evidence_run2_dum',\n",
       " 'rationality_external_evidence_edit_dum',\n",
       " 'rationality_background_info_edit_dum']"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset_w_pred_anon.columns.to_list()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "10bc45b2",
   "metadata": {},
   "outputs": [],
   "source": [
    "#rename old variabels by adding '_old'  ['interactivity_acknowledgement',  'rationality_external_evidence', 'rationality_topic_relevance', 'political_negativity', 'rationality_background_info', 'rationality_reasoning', 'political_ideology_US', 'rationality_reasoning_dum', 'rationality_background_info_dum', 'rationality_external_evidence_dum', 'rationality_prompt_dum', 'political_conservative_US', 'political_liberal_US', 'political_opinion_US', 'diversity_positiondum']:\n",
    "#dataset_w_pred_anon.rename(columns={'interactivity_acknowledgement': 'interactivity_acknowledgement_old', 'rationality_external_evidence': 'rationality_external_evidence_old', 'rationality_topic_relevance': 'rationality_topic_relevance_old', \n",
    "#                                    'political_negativity': 'political_negativity_old', 'rationality_background_info': 'rationality_background_info_old', 'rationality_reasoning': 'rationality_reasoning_old', 'political_ideology_US': 'political_ideology_US_old', \n",
    "#                                    'rationality_reasoning_dum': 'rationality_reasoning_dum_old', 'rationality_background_info_dum': 'rationality_background_info_dum_old', 'rationality_external_evidence_dum': 'rationality_external_evidence_dum_old', \n",
    "#                                    'rationality_prompt_dum': 'rationality_prompt_dum_old', 'political_conservative_US': 'political_conservative_US_old', 'political_liberal_US' : 'political_liberal_US_old', 'political_opinion_US': 'political_opinion_US_old', \n",
    "#                                    'diversity_positiondum': 'diversity_positiondum_old', 'diversity_positiondum_dum': 'diversity_positiondum_dum_old' }, inplace=True)\n",
    "#rename 'incivility_namecalling_dum' and 'incivility_namecalling' to indicate they have missings:\n",
    "#dataset_w_pred_anon.rename(columns={'incivility_namecalling_dum': 'incivility_namecalling_dum_NA', 'incivility_namecalling': 'incivility_namecalling_NA'}, inplace=True)                                    \n",
    "#rename 'incivility_namecalling_dum_low2' as standard:\n",
    "#dataset_w_pred_anon.rename(columns={'incivility_namecalling_low2': 'incivility_namecalling'}, inplace=True) \n",
    "#rename variables to indicate they are a duplicate run of the same variable:\n",
    "#dataset_w_pred_anon.rename(columns={'rationality_background_info': 'rationality_background_info_run2'}, inplace=True)\n",
    "#dataset_w_pred_anon.rename(columns={'rationality_external_evidence': 'rationality_external_evidence_run2'}, inplace=True)\n",
    "#rename variables to indicate they are an updated prompt version of the same variable:\n",
    "#dataset_w_pred_anon.rename(columns={'interactivity_acknowledgement': 'interactivity_acknowledgement_updated'}, inplace=True)\n",
    "\n",
    "\n",
    "###Note: the _edit-variables are the newest, followed by the _updated and not suffixed ones, and the _old ones are the oldest.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "49d2138c",
   "metadata": {},
   "outputs": [],
   "source": [
    "#make dummy variables for the items that are not yet dummified:\n",
    "dataset_w_pred_anon.loc[:, 'interactivity_acknowledgement_dum'] = dataset_w_pred_anon.loc[:, 'interactivity_acknowledgement_low'].map({\"Yes\": 1, \"No\":0}).fillna(0).astype(int)\n",
    "dataset_w_pred_anon.loc[:, 'rationality_external_evidence_dum'] = dataset_w_pred_anon.loc[:, 'rationality_external_evidence_low'].map({\"Yes\": 1, \"No\":0}).fillna(0).astype(int)\n",
    "dataset_w_pred_anon.loc[:, 'rationality_topic_relevance_dum'] = dataset_w_pred_anon.loc[:, 'rationality_topic_relevance_low'].map({\"Yes\": 1, \"No\":0}).fillna(0).astype(int)\n",
    "dataset_w_pred_anon.loc[:, 'rationality_background_info_low_dum'] = dataset_w_pred_anon.loc[:, 'rationality_background_info_low'].map({\"Yes\": 1, \"No\":0}).fillna(0).astype(int)\n",
    "dataset_w_pred_anon.loc[:, 'rationality_reasoning_dum'] = dataset_w_pred_anon.loc[:, 'rationality_reasoning_low'].map({\"Yes\": 1, \"No\":0}).fillna(0).astype(int)\n",
    "dataset_w_pred_anon.loc[:, 'intolerance_discrimination_dum'] = dataset_w_pred_anon.loc[:, 'intolerance_discrimination'].map({\"Yes\": 1, \"No\":0}).fillna(0).astype(int)\n",
    "dataset_w_pred_anon.loc[:, 'intolerance_rights_dum'] = dataset_w_pred_anon.loc[:, 'intolerance_rights'].map({\"Yes\": 1, \"No\":0}).fillna(0).astype(int)   \n",
    "dataset_w_pred_anon.loc[:, 'intolerance_violence_dum'] = dataset_w_pred_anon.loc[:, 'intolerance_violence'].map({\"Yes\": 1, \"No\":0}).fillna(0).astype(int)\n",
    "dataset_w_pred_anon.loc[:, 'incivility_namecalling_dum'] = dataset_w_pred_anon.loc[:, 'incivility_namecalling'].map({\"Yes\": 1, \"No\":0}).fillna(0).astype(int)\n",
    "\n",
    "dataset_w_pred_anon.loc[:, 'political_conservative_US_dum'] = dataset_w_pred_anon.loc[:, 'political_ideology_US_low'].map({\"conservative\": 1}).fillna(0).astype(int)\n",
    "dataset_w_pred_anon.loc[:, 'political_liberal_US_dum'] = dataset_w_pred_anon.loc[:, 'political_ideology_US_low'].map({\"liberal\": 1}).fillna(0).astype(int)\n",
    "dataset_w_pred_anon.loc[:, 'political_opinion_US_dum'] = dataset_w_pred_anon.loc[:, 'political_ideology_US_low'].map({\"liberal\": 1, 'conservative': 1}).fillna(0).astype(int)\n",
    "\n",
    "dataset_w_pred_anon.loc[:, 'diversity_conservative_dum'] = dataset_w_pred_anon.loc[:, 'diversity_ideologicaldirection'].map({\"Left/Liberal/Democratic\": 0, \"Absent\":0, \"Neutral\":0, \"Unclear which direction\":0 , \"Right/Conservative/Republican\":1}).fillna(0).astype(int)\n",
    "dataset_w_pred_anon.loc[:, 'diversity_liberal_dum'] = dataset_w_pred_anon.loc[:, 'diversity_ideologicaldirection'].map({\"Left/Liberal/Democratic\": 1, \"Absent\":0, \"Neutral\":0, \"Unclear which direction\":0 , \"Right/Conservative/Republican\":0}).fillna(0).astype(int)\n",
    "\n",
    "dataset_w_pred_anon.loc[:, 'interactivity_acknowledgement_updated_dum'] = dataset_w_pred_anon.loc[:, 'interactivity_acknowledgement_updated'].map({\"Yes\": 1, \"No\":0}).fillna(0).astype(int)\n",
    "dataset_w_pred_anon.loc[:, 'rationality_background_info_updated_dum'] = dataset_w_pred_anon.loc[:, 'rationality_background_info_updated'].map({\"Yes\": 1, \"No\":0}).fillna(0).astype(int)\n",
    "dataset_w_pred_anon.loc[:, 'rationality_external_evidence_updated_dum'] = dataset_w_pred_anon.loc[:, 'rationality_external_evidence_updated'].map({\"Yes\": 1, \"No\":0}).fillna(0).astype(int)\n",
    "dataset_w_pred_anon.loc[:, 'incivility_namecalling_updated_dum'] = dataset_w_pred_anon.loc[:, 'incivility_namecalling_updated'].map({\"Yes\": 1, \"No\":0}).fillna(0).astype(int)\n",
    "\n",
    "dataset_w_pred_anon.loc[:, 'incivility_namecalling_edit_dum'] = dataset_w_pred_anon.loc[:, 'incivility_namecalling_edit'].map({\"Yes\": 1, \"No\":0}).fillna(0).astype(int)\n",
    "dataset_w_pred_anon.loc[:, 'rationality_reasoning_edit_dum'] = dataset_w_pred_anon.loc[:, 'rationality_reasoning_edit'].map({\"Yes\": 1, \"No\":0}).fillna(0).astype(int)\n",
    "dataset_w_pred_anon.loc[:, 'rationality_background_info_run2_dum'] = dataset_w_pred_anon.loc[:, 'rationality_background_info_run2'].map({\"Yes\": 1, \"No\":0}).fillna(0).astype(int)\n",
    "dataset_w_pred_anon.loc[:, 'rationality_external_evidence_run2_dum'] = dataset_w_pred_anon.loc[:, 'rationality_external_evidence_run2'].map({\"Yes\": 1, \"No\":0}).fillna(0).astype(int)\n",
    "\n",
    "dataset_w_pred_anon.loc[:, 'rationality_external_evidence_edit_dum'] = dataset_w_pred_anon.loc[:, 'rationality_external_evidence_edit'].map({\"Yes\": 1, \"No\":0}).fillna(0).astype(int)\n",
    "dataset_w_pred_anon.loc[:, 'rationality_background_info_edit_dum'] = dataset_w_pred_anon.loc[:, 'rationality_background_info_edit'].map({\"Yes\": 1, \"No\":0}).fillna(0).astype(int)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "5b6d2548",
   "metadata": {},
   "outputs": [],
   "source": [
    "#create compound-concept variables with the latest version of the prompt:\n",
    "dataset_w_pred_anon.loc[:, 'rationality_prompt_dum'] = dataset_w_pred_anon.loc[:, ['rationality_reasoning_edit_dum','rationality_background_info_edit_dum','rationality_external_evidence_edit_dum']].max(axis=1)\n",
    "dataset_w_pred_anon.loc[:, 'incivility_prompt_dum'] = dataset_w_pred_anon.loc[:, ['incivility_namecalling_edit_dum', 'incivility_vulgarity_dum', 'incivility_attackreputation_dum', \n",
    "                'incivility_question_intelligence_dum', 'incivility_allcaps_dum', 'incivility_sarcasm_dum',\n",
    "              'intolerance_rights_dum', 'intolerance_discrimination_dum', 'intolerance_violence_dum']].max(axis=1)\n",
    "dataset_w_pred_anon.loc[:, 'hate_list_prompt_dum'] = dataset_w_pred_anon.loc[:, ['incivility_namecalling_edit_dum', 'incivility_vulgarity_dum', 'intolerance_rights_dum', 'intolerance_discrimination_dum', 'intolerance_violence_dum']].max(axis=1) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "adbfed7f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>rationality_prompt_dum</th>\n",
       "      <th>rationality_reasoning_dum</th>\n",
       "      <th>rationality_background_info_low_dum</th>\n",
       "      <th>rationality_external_evidence_dum</th>\n",
       "      <th>rationality_external_evidence_low</th>\n",
       "      <th>rationality_background_info_low</th>\n",
       "      <th>rationality_reasoning_low</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>None</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>None</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>None</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   rationality_prompt_dum  rationality_reasoning_dum  \\\n",
       "0                       0                          0   \n",
       "1                       1                          1   \n",
       "2                       1                          1   \n",
       "3                       1                          1   \n",
       "4                       1                          1   \n",
       "5                       0                          0   \n",
       "6                       0                          0   \n",
       "7                       0                          0   \n",
       "8                       1                          1   \n",
       "9                       1                          1   \n",
       "\n",
       "   rationality_background_info_low_dum  rationality_external_evidence_dum  \\\n",
       "0                                    0                                  0   \n",
       "1                                    0                                  0   \n",
       "2                                    1                                  1   \n",
       "3                                    0                                  0   \n",
       "4                                    0                                  0   \n",
       "5                                    0                                  0   \n",
       "6                                    0                                  0   \n",
       "7                                    0                                  0   \n",
       "8                                    0                                  0   \n",
       "9                                    0                                  0   \n",
       "\n",
       "  rationality_external_evidence_low rationality_background_info_low  \\\n",
       "0                                No                              No   \n",
       "1                                No                              No   \n",
       "2                               Yes                             Yes   \n",
       "3                                No                              No   \n",
       "4                                No                              No   \n",
       "5                              None                              No   \n",
       "6                              None                              No   \n",
       "7                              None                              No   \n",
       "8                                No                              No   \n",
       "9                                No                              No   \n",
       "\n",
       "  rationality_reasoning_low  \n",
       "0                        No  \n",
       "1                       Yes  \n",
       "2                       Yes  \n",
       "3                       Yes  \n",
       "4                       Yes  \n",
       "5                        No  \n",
       "6                        No  \n",
       "7                        No  \n",
       "8                       Yes  \n",
       "9                       Yes  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset_w_pred_anon.loc[:, ['rationality_prompt_dum', 'rationality_reasoning_dum','rationality_background_info_low_dum','rationality_external_evidence_dum', 'rationality_external_evidence_low','rationality_background_info_low','rationality_reasoning_low' ]].head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "755e90b0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "interactivity_acknowledgement_low           2\n",
       "rationality_external_evidence_low         910\n",
       "rationality_background_info_low            90\n",
       "rationality_reasoning_low                 102\n",
       "political_ideology_US_low                   0\n",
       "rationality_topic_relevance_low             2\n",
       "political_ideology_US_low                   0\n",
       "incivility_namecalling                    156\n",
       "incivility_namecalling_NA                 174\n",
       "incivility_vulgarity                       16\n",
       "incivility_attackreputation                 9\n",
       "incivility_question_intelligence            3\n",
       "incivility_allcaps                          5\n",
       "incivility_sarcasm                          6\n",
       "intolerance_rights                          3\n",
       "intolerance_discrimination                  1\n",
       "intolerance_violence                        3\n",
       "diversity_ideologicaldirection              5\n",
       "diversity_position                          3\n",
       "diversity_disagreement                      4\n",
       "rationality_background_info_updated       151\n",
       "rationality_external_evidence_updated    1354\n",
       "interactivity_acknowledgement_updated       2\n",
       "incivility_namecalling_updated              3\n",
       "incivility_namecalling_edit                 5\n",
       "rationality_reasoning_edit                  4\n",
       "rationality_background_info_run2          142\n",
       "rationality_external_evidence_run2       1351\n",
       "rationality_external_evidence_edit          2\n",
       "rationality_background_info_edit            4\n",
       "dtype: int64"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#count NA's:\n",
    "dataset_w_pred_anon.loc[:, ['interactivity_acknowledgement_low', 'rationality_external_evidence_low','rationality_background_info_low','rationality_reasoning_low', 'political_ideology_US_low', \"rationality_topic_relevance_low\", \n",
    "                            \"political_ideology_US_low\", 'incivility_namecalling', 'incivility_namecalling_NA', 'incivility_vulgarity', 'incivility_attackreputation', \n",
    "                            'incivility_question_intelligence', 'incivility_allcaps', 'incivility_sarcasm', 'intolerance_rights', 'intolerance_discrimination', 'intolerance_violence', \n",
    "                            'diversity_ideologicaldirection', 'diversity_position', 'diversity_disagreement', 'rationality_background_info_updated', 'rationality_external_evidence_updated',\n",
    "                            'interactivity_acknowledgement_updated', 'incivility_namecalling_updated',\n",
    "                            'incivility_namecalling_edit','rationality_reasoning_edit', 'rationality_background_info_run2','rationality_external_evidence_run2', 'rationality_external_evidence_edit', \n",
    "                            'rationality_background_info_edit']].isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "07f8f6f1",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Note many missing values...especially in external_evidence.\n",
    "#but edited prompts fixed the issue, so we can use the edited prompts for the analysis."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "67800c00",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "rationality_background_info_run2       No  Yes  NaN     All\n",
      "rationality_background_info_updated                        \n",
      "No                                   2690   10   21  2721.0\n",
      "Yes                                    11  979    0   990.0\n",
      "NaN                                    30    0  121     NaN\n",
      "All                                  2731  989    0  3862.0\n",
      "---\n",
      "rationality_external_evidence_run2       No  Yes   NaN     All\n",
      "rationality_external_evidence_updated                         \n",
      "No                                     1615   11    85  1711.0\n",
      "Yes                                       9  788     0   797.0\n",
      "NaN                                      86    2  1266     NaN\n",
      "All                                    1710  801     0  3862.0\n",
      "---\n",
      "effect of temperature:\n",
      "rationality_background_info_low    No   Yes  NaN     All\n",
      "rationality_background_info_old                         \n",
      "No                               2728   343   87  3158.0\n",
      "Yes                                33   667    0   700.0\n",
      "NaN                                 0     1    3     NaN\n",
      "All                              2761  1011    0  3862.0\n",
      "rationality_external_evidence_low    No   Yes  NaN     All\n",
      "rationality_external_evidence_old                         \n",
      "No                                 1884   465  902  3251.0\n",
      "Yes                                  34   561    5   600.0\n",
      "NaN                                   0     8    3     NaN\n",
      "All                                1918  1034    0  3862.0\n"
     ]
    }
   ],
   "source": [
    "#check run rerun accuracy:\n",
    "print(pd.crosstab(dataset_w_pred_anon['rationality_background_info_updated'], dataset_w_pred_anon['rationality_background_info_run2'], margins=True, dropna=False))\n",
    "print(\"---\")\n",
    "print(pd.crosstab(dataset_w_pred_anon['rationality_external_evidence_updated'], dataset_w_pred_anon['rationality_external_evidence_run2'], margins=True, dropna=False))\n",
    "\n",
    "#compare to runs with and without seed/low temperature:\n",
    "print(\"---\")\n",
    "print(\"effect of temperature:\")\n",
    "print(pd.crosstab(dataset_w_pred_anon['rationality_background_info_old'], dataset_w_pred_anon['rationality_background_info_low'], margins=True, dropna=False))\n",
    "print(pd.crosstab(dataset_w_pred_anon['rationality_external_evidence_old'], dataset_w_pred_anon['rationality_external_evidence_low'], margins=True, dropna=False))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "937b8e14",
   "metadata": {},
   "source": [
    "much less difference in run-rerun within seed and low temperature, than in high-low temperature between seeds\n",
    "Still setting the seed and use low temperature, did not produce fully identical results...\n",
    "We do observe a strong increase in NAs for these variables with the change in seed an temperature, unexpectedly these were far few for the high temperature-version, perhaps a lucky random seed on that run?\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "2f1e3dca",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "rationality_background_info_edit       No  Yes  NaN     All\n",
      "rationality_background_info_updated                        \n",
      "No                                   2640   81    0  2721.0\n",
      "Yes                                    98  892    0   990.0\n",
      "NaN                                   147    0    4     NaN\n",
      "All                                  2885  973    0  3862.0\n",
      "---\n",
      "rationality_external_evidence_edit       No  Yes  NaN     All\n",
      "rationality_external_evidence_updated                        \n",
      "No                                     1582  129    0  1711.0\n",
      "Yes                                      56  740    1   797.0\n",
      "NaN                                    1337   16    1     NaN\n",
      "All                                    2975  885    0  3862.0\n",
      "---\n",
      "incivility_namecalling_edit       No   Yes  NaN     All\n",
      "incivility_namecalling_updated                         \n",
      "No                              2462   206    2  2670.0\n",
      "Yes                               23  1164    2  1189.0\n",
      "NaN                                1     1    1     NaN\n",
      "All                             2486  1371    0  3862.0\n",
      "---\n",
      "incivility_namecalling_edit    No   Yes  NaN     All\n",
      "incivility_namecalling                              \n",
      "No                           2313   221    0  2534.0\n",
      "Yes                            28  1143    1  1172.0\n",
      "NaN                           145     7    4     NaN\n",
      "All                          2486  1371    0  3862.0\n",
      "---\n",
      "rationality_reasoning_edit    No   Yes  NaN     All\n",
      "rationality_reasoning_low                          \n",
      "No                          1530    26    0  1556.0\n",
      "Yes                          359  1844    1  2204.0\n",
      "NaN                           99     0    3     NaN\n",
      "All                         1988  1870    0  3862.0\n"
     ]
    }
   ],
   "source": [
    "#what is the effect of reformulating the prompt:\n",
    "print(pd.crosstab(dataset_w_pred_anon['rationality_background_info_updated'], dataset_w_pred_anon['rationality_background_info_edit'], margins=True, dropna=False))\n",
    "print(\"---\")\n",
    "print(pd.crosstab(dataset_w_pred_anon['rationality_external_evidence_updated'], dataset_w_pred_anon['rationality_external_evidence_edit'], margins=True, dropna=False))\n",
    "print(\"---\")\n",
    "print(pd.crosstab(dataset_w_pred_anon['incivility_namecalling_updated'], dataset_w_pred_anon['incivility_namecalling_edit'], margins=True, dropna=False))\n",
    "print(\"---\")\n",
    "print(pd.crosstab(dataset_w_pred_anon['incivility_namecalling'], dataset_w_pred_anon['incivility_namecalling_edit'], margins=True, dropna=False))\n",
    "print(\"---\")\n",
    "print(pd.crosstab(dataset_w_pred_anon['rationality_reasoning_low'], dataset_w_pred_anon['rationality_reasoning_edit'], margins=True, dropna=False))\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ca480742",
   "metadata": {},
   "source": [
    "#not only effect on reduction of NAs but also quite some recoding to switch categories from yes -> no and vice versa, not as much as changing seed and temperature, but more than a simple rerun,\n",
    "#also nearly all NAs were now coded as 'NO' which is equivalent to the legacy handling of these values, so these do not change the results of the classifier performance statistics, improvements are largely due to switching labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6ef4c3c0",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "classifying diversity_disagreement_edit: 100%|██████████| 3862/3862 [28:09<00:00,  2.29it/s]  \n"
     ]
    }
   ],
   "source": [
    "#does this also hold for diversity_disagreement?\n",
    "pubspherepromptsrun16 = ['diversity_disagreement_edit']   \n",
    "predictions16: typing.Dict[str, np.ndarray] = {\n",
    "    label: (\n",
    "        src.PromptClassify\n",
    "        .from_json(path)\n",
    "        (dataset[\"commentText\"], model=MODELlarge, options=options_low)\n",
    "    )\n",
    "    for label, path in CFG.prompt_classify_files.items() if label in pubspherepromptsrun16\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "d80eb532",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "diversity_disagreement_edit\n",
      "Yes    2068\n",
      "No     1791\n",
      "Name: count, dtype: int64\n",
      "------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "#join to the dataset:   \n",
    "for _, preds in predictions16.items():\n",
    "    print(preds.value_counts())\n",
    "    print(\"-\" * 42)\n",
    "    dataset_w_pred_2 = dataset_w_pred_2.join(preds, rsuffix='_edit')\n",
    "    dataset_w_pred_anon = dataset_w_pred_anon.join(preds, rsuffix='_edit')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "id": "7553edab",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_w_pred_anon.loc[:, 'diversity_disagreement_edit_dum'] = dataset_w_pred_2.loc[:, 'diversity_disagreement_edit'].map({\"Yes\": 1, \"No\":0}).fillna(0).astype(int)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "id": "9e6c32d0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "diversity_disagreement_edit    No   Yes  NaN     All\n",
      "diversity_disagreement                              \n",
      "No                           1741   474    1  2216.0\n",
      "Yes                            49  1593    0  1642.0\n",
      "NaN                             1     1    2     NaN\n",
      "All                          1791  2068    0  3862.0\n"
     ]
    }
   ],
   "source": [
    "#compare results:\n",
    "print(pd.crosstab(dataset_w_pred_anon['diversity_disagreement'], dataset_w_pred_anon['diversity_disagreement_edit'], margins=True, dropna=False))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "id": "37af970e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "diversity_disagreement_edit    No   Yes  NaN   All\n",
      "disagreement                                      \n",
      "0                            1780  1896    3  3679\n",
      "1                              11   172    0   183\n",
      "All                          1791  2068    0  3862\n"
     ]
    }
   ],
   "source": [
    "#compare results to manual coding:\n",
    "print(pd.crosstab(dataset_w_pred_anon['disagreement'], dataset_w_pred_anon['diversity_disagreement_edit'], margins=True, dropna=False))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fdd5f89f",
   "metadata": {},
   "source": [
    "It appears the model finds vastly more disagreement, probably since it misses the context in the prompt to determine whether a post is an answer to a question, or real disagreement (since the thread is not provided in the prompt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "id": "7f428de4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['StartDate',\n",
       " 'RecordedDate',\n",
       " 'IPAddress',\n",
       " 'Finished',\n",
       " 'Coder',\n",
       " 'ID',\n",
       " 'Mark_ID',\n",
       " 'Genre',\n",
       " 'topiccode',\n",
       " 'Platform',\n",
       " 'Anonymity',\n",
       " 'Anonymity_9_TEXT',\n",
       " 'codable',\n",
       " 'Interaction',\n",
       " 'Acknowledgement',\n",
       " 'TopicRelevance',\n",
       " 'Reasoning',\n",
       " 'BackgroundInfo',\n",
       " 'ExternalEvidence',\n",
       " 'ExternalEvidence_1_TEXT',\n",
       " 'Opinion',\n",
       " 'disagreement',\n",
       " 'Ideologicaldirection',\n",
       " 'Name_calling',\n",
       " 'Vulgarity',\n",
       " 'Attack_reputation',\n",
       " 'Question_Intelligenc',\n",
       " 'All_caps_function',\n",
       " 'Sarcasm_to_criticize',\n",
       " 'Individual_right',\n",
       " 'discrimination',\n",
       " 'Invoke_violence',\n",
       " 'Tone',\n",
       " 'INTERACTIVITY_DUMMY',\n",
       " 'RATIONALITY_DUMMY',\n",
       " 'HAS_OPINION_DUMMY',\n",
       " 'LIBERAL_NEUTRAL_CONSERVATIVE',\n",
       " 'LIBERAL_DUMMY',\n",
       " 'CONSERVATIVE_DUMMY',\n",
       " 'NAMECALLING_DUMMY',\n",
       " 'VULGAR_DUMMY',\n",
       " 'NAMECALLING_VULGAR_DUMMY',\n",
       " 'INCIVILITY_ORDINAL',\n",
       " 'INCIVILITY_DUMMY',\n",
       " 'INTOLERANCE_DUMMY',\n",
       " 'filter_$',\n",
       " 'IMPOLITENESS_DUMMY',\n",
       " 'showName',\n",
       " 'genre',\n",
       " 'Time_comment',\n",
       " 'likeCount_comment',\n",
       " 'entities',\n",
       " 'place',\n",
       " 'retweet_count',\n",
       " 'platform',\n",
       " 'retweeted',\n",
       " 'language',\n",
       " 'source',\n",
       " 'in_reply_to_status_id_str',\n",
       " 'in_reply_to_user_id_str',\n",
       " 'in_reply_to_screen_name',\n",
       " 'is_quote_status',\n",
       " 'videoTitle',\n",
       " 'description',\n",
       " 'Time_video',\n",
       " 'channelTitle',\n",
       " 'channelId',\n",
       " 'viewCount',\n",
       " 'dislikeCount_video',\n",
       " 'likeCount_video',\n",
       " 'date_difference',\n",
       " 'commentCount_video',\n",
       " 'replyCount_comment',\n",
       " 'topic',\n",
       " 'subscribers',\n",
       " 'HATELIST_FOCUSED_DUMMY',\n",
       " 'Time_comment_year',\n",
       " 'Time_video_year',\n",
       " 'interactivity_acknowledgement_old',\n",
       " 'rationality_external_evidence_old',\n",
       " 'rationality_topic_relevance_old',\n",
       " 'political_negativity_old',\n",
       " 'rationality_background_info_old',\n",
       " 'rationality_reasoning_old',\n",
       " 'topics',\n",
       " 'emotions',\n",
       " 'sentiment',\n",
       " 'irony',\n",
       " 'offensive',\n",
       " 'hate',\n",
       " 'political_ideology_US_old',\n",
       " 'offensive_dum',\n",
       " 'hate_dum',\n",
       " 'cardiff_incivil',\n",
       " 'rationality_reasoning_dum_old',\n",
       " 'rationality_background_info_dum_old',\n",
       " 'rationality_external_evidence_dum_old',\n",
       " 'rationality_prompt_dum_old',\n",
       " 'political_conservative_US_old',\n",
       " 'political_liberal_US_old',\n",
       " 'political_opinion_US_old',\n",
       " 'diversity_positiondum_old',\n",
       " 'diversity_disagreement',\n",
       " 'diversity_ideologicaldirection',\n",
       " 'diversity_position',\n",
       " 'incivility_allcaps',\n",
       " 'incivility_attackreputation',\n",
       " 'incivility_question_intelligence',\n",
       " 'incivility_sarcasm',\n",
       " 'incivility_namecalling_NA',\n",
       " 'incivility_vulgarity',\n",
       " 'diversity_positiondum_dum_old',\n",
       " 'diversity_disagreement_dum',\n",
       " 'diversity_position_dum',\n",
       " 'incivility_allcaps_dum',\n",
       " 'incivility_attackreputation_dum',\n",
       " 'incivility_question_intelligence_dum',\n",
       " 'incivility_sarcasm_dum',\n",
       " 'incivility_namecalling_dum_NA',\n",
       " 'incivility_vulgarity_dum',\n",
       " 'diversity_ideologicaldirection_dum',\n",
       " 'interactivity_acknowledgement_low',\n",
       " 'intolerance_discrimination',\n",
       " 'intolerance_rights',\n",
       " 'intolerance_violence',\n",
       " 'political_ideology_US_low',\n",
       " 'rationality_background_info_low',\n",
       " 'rationality_external_evidence_low',\n",
       " 'rationality_reasoning_low',\n",
       " 'rationality_topic_relevance_low',\n",
       " 'incivility_namecalling',\n",
       " 'rationality_background_info_updated',\n",
       " 'rationality_external_evidence_updated',\n",
       " 'interactivity_acknowledgement_dum',\n",
       " 'rationality_external_evidence_dum',\n",
       " 'rationality_topic_relevance_dum',\n",
       " 'rationality_background_info_low_dum',\n",
       " 'rationality_reasoning_dum',\n",
       " 'intolerance_discrimination_dum',\n",
       " 'intolerance_rights_dum',\n",
       " 'intolerance_violence_dum',\n",
       " 'incivility_namecalling_dum',\n",
       " 'political_conservative_US_dum',\n",
       " 'political_liberal_US_dum',\n",
       " 'political_opinion_US_dum',\n",
       " 'diversity_liberal_dum',\n",
       " 'rationality_background_info_updated_dum',\n",
       " 'rationality_external_evidence_updated_dum',\n",
       " 'rationality_prompt_dum',\n",
       " 'incivility_prompt_dum',\n",
       " 'hate_list_prompt_dum',\n",
       " 'rationality_background_info_run2',\n",
       " 'rationality_external_evidence_run2',\n",
       " 'rationality_external_evidence_edit',\n",
       " 'rationality_background_info_edit',\n",
       " 'interactivity_acknowledgement_updated',\n",
       " 'rationality_reasoning_edit',\n",
       " 'incivility_namecalling_updated',\n",
       " 'incivility_namecalling_edit',\n",
       " 'diversity_conservative_dum',\n",
       " 'interactivity_acknowledgement_updated_dum',\n",
       " 'incivility_namecalling_updated_dum',\n",
       " 'incivility_namecalling_edit_dum',\n",
       " 'rationality_reasoning_edit_dum',\n",
       " 'rationality_background_info_run2_dum',\n",
       " 'rationality_external_evidence_run2_dum',\n",
       " 'rationality_external_evidence_edit_dum',\n",
       " 'rationality_background_info_edit_dum',\n",
       " 'diversity_ideologicaldirection_long',\n",
       " 'diversity_conservative_long_dum',\n",
       " 'diversity_liberal_long_dum',\n",
       " 'diversity_disagreement_edit',\n",
       " 'diversity_disagreement_edit_dum']"
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset_w_pred_anon.columns.to_list()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "id": "897fe7c1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      0.48      0.65      3679\n",
      "           1       0.08      0.94      0.15       183\n",
      "\n",
      "    accuracy                           0.51      3862\n",
      "   macro avg       0.54      0.71      0.40      3862\n",
      "weighted avg       0.95      0.51      0.63      3862\n",
      "\n",
      "cohen_kappa_score(disagreement): 0.07202694798722953\n",
      "krippendorf(disagreement): -0.1954620245925982\n",
      "percent_agreement(disagreement): 50.62%\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      0.60      0.75      3679\n",
      "           1       0.11      0.95      0.19       183\n",
      "\n",
      "    accuracy                           0.62      3862\n",
      "   macro avg       0.55      0.78      0.47      3862\n",
      "weighted avg       0.95      0.62      0.72      3862\n",
      "\n",
      "cohen_kappa_score(disagreement): 0.11524484045577765\n",
      "krippendorf(disagreement): -0.059559293773962185\n",
      "percent_agreement(disagreement): 61.76%\n"
     ]
    }
   ],
   "source": [
    "#evaluete respective performance:\n",
    "llm_human_column_pairs = [\n",
    "    (\"disagreement\", \"diversity_disagreement_edit_dum\"),\n",
    "    (\"disagreement\", \"diversity_disagreement_dum\")\n",
    "]\n",
    "for human_col, llm_col in llm_human_column_pairs:\n",
    "    subset = dataset_w_pred_anon[[human_col, llm_col]].dropna()\n",
    "    human = subset[human_col].tolist()\n",
    "    llm = subset[llm_col].tolist()\n",
    "\n",
    "    # Calculate percent agreement\n",
    "    agreement = np.sum(np.array(human) == np.array(llm))/len(np.array(human)) * 100\n",
    "\n",
    "    print(\"---\")\n",
    "    print(f\"{classification_report(human, llm)}\")\n",
    "    print(f\"cohen_kappa_score({human_col}): {cohen_kappa_score(human, llm)}\")\n",
    "    print(f\"krippendorf({human_col}): {krippendorff.alpha(np.array([human, llm]), level_of_measurement=\"nominal\")}\")\n",
    "    print(f\"percent_agreement({human_col}): {agreement:.2f}%\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e4546316",
   "metadata": {},
   "source": [
    "The results of the edited prompt are even worse than the original prompt"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "23d7b549",
   "metadata": {},
   "source": [
    "perhaps this is because diversity_disagreement— only applies if answer to this option was \"Yes\":       __Does this comment refer to a societally or __politically relevant issue (i.e. on-topic)?\n",
    "So maybe we should only look at those comments for results\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "87be6a10",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "disagreement       0    1   All\n",
      "TopicRelevance                 \n",
      "0               1498   53  1551\n",
      "1               2181  130  2311\n",
      "All             3679  183  3862\n"
     ]
    }
   ],
   "source": [
    "print(pd.crosstab(dataset_w_pred_anon['TopicRelevance'], dataset_w_pred_anon['disagreement'], margins=True, dropna=False))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8d919360",
   "metadata": {},
   "source": [
    "no, coding doesn't appear to be restricted to topicrelevant comments..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "01fed1f6",
   "metadata": {},
   "outputs": [],
   "source": [
    "#evaluate performance against manual coding of rationality_prompt_dum incivility_prompt_dum hate_list_prompt_dum interactivity_acknowledgement_dum, political_conservative_US_dum, political_liberal_US_dum, political_opinion_US_dum:\n",
    "llm_human_column_pairs = [\n",
    "    (\"LIBERAL_DUMMY\", \"political_liberal_US_dum\"),\n",
    "    (\"CONSERVATIVE_DUMMY\", \"political_conservative_US_dum\"),\n",
    "    (\"HAS_OPINION_DUMMY\", \"political_opinion_US_dum\"),\n",
    "    (\"LIBERAL_DUMMY\", \"diversity_liberal_dum\"),\n",
    "    (\"CONSERVATIVE_DUMMY\", \"diversity_conservative_dum\"),\n",
    "    (\"HAS_OPINION_DUMMY\", \"diversity_position_dum\"),\n",
    "    (\"RATIONALITY_DUMMY\", \"rationality_prompt_dum\"),\n",
    "    (\"TopicRelevance\", \"rationality_topic_relevance_dum\"),\n",
    "    (\"INTERACTIVITY_DUMMY\", \"interactivity_acknowledgement_updated_dum\"),\n",
    "    (\"INCIVILITY_DUMMY\", \"incivility_prompt_dum\"),\n",
    "    (\"HATELIST_FOCUSED_DUMMY\", \"hate_list_prompt_dum\")\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "94f58c02",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "split\n",
       "train    3089\n",
       "test      773\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset_w_pred_anon.loc[:, 'split'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "ed58c723",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Inter-coder reliablity: classification_report(human as gold, llm as predicted), cohen_kappa(human, llm):\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.93      0.85      0.89      2440\n",
      "           1       0.57      0.77      0.65       649\n",
      "\n",
      "    accuracy                           0.83      3089\n",
      "   macro avg       0.75      0.81      0.77      3089\n",
      "weighted avg       0.86      0.83      0.84      3089\n",
      "\n",
      "cohen_kappa_score(LIBERAL_DUMMY): 0.5435589327727111\n",
      "krippendorf(LIBERAL_DUMMY): 0.5404288426702988\n",
      "percent_agreement(LIBERAL_DUMMY): 82.94%\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.95      0.90      0.92      2611\n",
      "           1       0.57      0.73      0.64       478\n",
      "\n",
      "    accuracy                           0.87      3089\n",
      "   macro avg       0.76      0.81      0.78      3089\n",
      "weighted avg       0.89      0.87      0.88      3089\n",
      "\n",
      "cohen_kappa_score(CONSERVATIVE_DUMMY): 0.5615336292118079\n",
      "krippendorf(CONSERVATIVE_DUMMY): 0.5600409196001979\n",
      "percent_agreement(CONSERVATIVE_DUMMY): 87.15%\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.77      0.81      0.79      1505\n",
      "           1       0.81      0.76      0.79      1584\n",
      "\n",
      "    accuracy                           0.79      3089\n",
      "   macro avg       0.79      0.79      0.79      3089\n",
      "weighted avg       0.79      0.79      0.79      3089\n",
      "\n",
      "cohen_kappa_score(HAS_OPINION_DUMMY): 0.5749961205707755\n",
      "krippendorf(HAS_OPINION_DUMMY): 0.5746797355824025\n",
      "percent_agreement(HAS_OPINION_DUMMY): 78.73%\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.89      0.90      0.89      2440\n",
      "           1       0.60      0.59      0.60       649\n",
      "\n",
      "    accuracy                           0.83      3089\n",
      "   macro avg       0.75      0.74      0.75      3089\n",
      "weighted avg       0.83      0.83      0.83      3089\n",
      "\n",
      "cohen_kappa_score(LIBERAL_DUMMY): 0.4925959969104845\n",
      "krippendorf(LIBERAL_DUMMY): 0.49266837571887667\n",
      "percent_agreement(LIBERAL_DUMMY): 83.26%\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.96      0.77      0.86      2611\n",
      "           1       0.40      0.83      0.54       478\n",
      "\n",
      "    accuracy                           0.78      3089\n",
      "   macro avg       0.68      0.80      0.70      3089\n",
      "weighted avg       0.87      0.78      0.81      3089\n",
      "\n",
      "cohen_kappa_score(CONSERVATIVE_DUMMY): 0.4178716197635107\n",
      "krippendorf(CONSERVATIVE_DUMMY): 0.3961260300280468\n",
      "percent_agreement(CONSERVATIVE_DUMMY): 78.15%\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.93      0.54      0.69      1505\n",
      "           1       0.69      0.96      0.80      1584\n",
      "\n",
      "    accuracy                           0.76      3089\n",
      "   macro avg       0.81      0.75      0.74      3089\n",
      "weighted avg       0.81      0.76      0.75      3089\n",
      "\n",
      "cohen_kappa_score(HAS_OPINION_DUMMY): 0.5096252788964486\n",
      "krippendorf(HAS_OPINION_DUMMY): 0.4882481439125498\n",
      "percent_agreement(HAS_OPINION_DUMMY): 75.75%\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.97      0.57      0.72      2541\n",
      "           1       0.32      0.93      0.47       548\n",
      "\n",
      "    accuracy                           0.64      3089\n",
      "   macro avg       0.65      0.75      0.60      3089\n",
      "weighted avg       0.86      0.64      0.68      3089\n",
      "\n",
      "cohen_kappa_score(RATIONALITY_DUMMY): 0.2860732172862267\n",
      "krippendorf(RATIONALITY_DUMMY): 0.19600248379741014\n",
      "percent_agreement(RATIONALITY_DUMMY): 63.58%\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.84      0.70      0.76      1233\n",
      "           1       0.82      0.91      0.86      1856\n",
      "\n",
      "    accuracy                           0.82      3089\n",
      "   macro avg       0.83      0.80      0.81      3089\n",
      "weighted avg       0.83      0.82      0.82      3089\n",
      "\n",
      "cohen_kappa_score(TopicRelevance): 0.6237501696703944\n",
      "krippendorf(TopicRelevance): 0.6220436866504282\n",
      "percent_agreement(TopicRelevance): 82.45%\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.85      0.71      0.77      2297\n",
      "           1       0.42      0.62      0.50       792\n",
      "\n",
      "    accuracy                           0.69      3089\n",
      "   macro avg       0.63      0.67      0.64      3089\n",
      "weighted avg       0.74      0.69      0.70      3089\n",
      "\n",
      "cohen_kappa_score(INTERACTIVITY_DUMMY): 0.28588521833672575\n",
      "krippendorf(INTERACTIVITY_DUMMY): 0.2736054735017758\n",
      "percent_agreement(INTERACTIVITY_DUMMY): 68.50%\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.90      0.47      0.62      1567\n",
      "           1       0.63      0.94      0.76      1522\n",
      "\n",
      "    accuracy                           0.70      3089\n",
      "   macro avg       0.77      0.71      0.69      3089\n",
      "weighted avg       0.77      0.70      0.69      3089\n",
      "\n",
      "cohen_kappa_score(INCIVILITY_DUMMY): 0.41032106190851014\n",
      "krippendorf(INCIVILITY_DUMMY): 0.3738538552480819\n",
      "percent_agreement(INCIVILITY_DUMMY): 70.31%\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.97      0.69      0.81      2391\n",
      "           1       0.47      0.92      0.62       698\n",
      "\n",
      "    accuracy                           0.75      3089\n",
      "   macro avg       0.72      0.81      0.71      3089\n",
      "weighted avg       0.86      0.75      0.77      3089\n",
      "\n",
      "cohen_kappa_score(HATELIST_FOCUSED_DUMMY): 0.45865774881763\n",
      "krippendorf(HATELIST_FOCUSED_DUMMY): 0.42927608928949423\n",
      "percent_agreement(HATELIST_FOCUSED_DUMMY): 74.52%\n"
     ]
    }
   ],
   "source": [
    "#first evaluate performance against manual coding in the training set:\n",
    "\n",
    "#these are the new results of the edited prompts:\n",
    "\n",
    "print(\"Inter-coder reliablity: classification_report(human as gold, llm as predicted), cohen_kappa(human, llm):\")\n",
    "\n",
    "for human_col, llm_col in llm_human_column_pairs:\n",
    "    subset = dataset_w_pred_anon.loc[dataset_w_pred_anon.split == 'train', [human_col, llm_col]].dropna()\n",
    "    human = subset[human_col].tolist()\n",
    "    llm = subset[llm_col].tolist()\n",
    "\n",
    "    # Calculate percent agreement\n",
    "    agreement = np.sum(np.array(human) == np.array(llm))/len(np.array(human)) * 100\n",
    "\n",
    "    print(\"---\")\n",
    "    print(f\"{classification_report(human, llm)}\")\n",
    "    print(f\"cohen_kappa_score({human_col}): {cohen_kappa_score(human, llm)}\")\n",
    "    print(f\"krippendorf({human_col}): {krippendorff.alpha(np.array([human, llm]), level_of_measurement=\"nominal\")}\")\n",
    "    print(f\"percent_agreement({human_col}): {agreement:.2f}%\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5083e945",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Inter-coder reliablity: classification_report(human as gold, llm as predicted), cohen_kappa(human, llm):\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.94      0.85      0.89      3073\n",
      "           1       0.56      0.77      0.65       789\n",
      "\n",
      "    accuracy                           0.83      3862\n",
      "   macro avg       0.75      0.81      0.77      3862\n",
      "weighted avg       0.86      0.83      0.84      3862\n",
      "\n",
      "cohen_kappa_score(LIBERAL_DUMMY): 0.5444043080556233\n",
      "krippendorf(LIBERAL_DUMMY): 0.5408921571554941\n",
      "percent_agreement(LIBERAL_DUMMY): 83.14%\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.95      0.90      0.92      3271\n",
      "           1       0.57      0.74      0.65       591\n",
      "\n",
      "    accuracy                           0.88      3862\n",
      "   macro avg       0.76      0.82      0.79      3862\n",
      "weighted avg       0.89      0.88      0.88      3862\n",
      "\n",
      "cohen_kappa_score(CONSERVATIVE_DUMMY): 0.5725715490094205\n",
      "krippendorf(CONSERVATIVE_DUMMY): 0.571045663399131\n",
      "percent_agreement(CONSERVATIVE_DUMMY): 87.55%\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.77      0.82      0.79      1907\n",
      "           1       0.81      0.77      0.79      1955\n",
      "\n",
      "    accuracy                           0.79      3862\n",
      "   macro avg       0.79      0.79      0.79      3862\n",
      "weighted avg       0.79      0.79      0.79      3862\n",
      "\n",
      "cohen_kappa_score(HAS_OPINION_DUMMY): 0.5843646306803332\n",
      "krippendorf(HAS_OPINION_DUMMY): 0.5841227804229595\n",
      "percent_agreement(HAS_OPINION_DUMMY): 79.21%\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.90      0.90      0.90      3073\n",
      "           1       0.60      0.60      0.60       789\n",
      "\n",
      "    accuracy                           0.84      3862\n",
      "   macro avg       0.75      0.75      0.75      3862\n",
      "weighted avg       0.84      0.84      0.84      3862\n",
      "\n",
      "cohen_kappa_score(LIBERAL_DUMMY): 0.49554428091035374\n",
      "krippendorf(LIBERAL_DUMMY): 0.49560938324103376\n",
      "percent_agreement(LIBERAL_DUMMY): 83.58%\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.96      0.77      0.85      3271\n",
      "           1       0.39      0.82      0.53       591\n",
      "\n",
      "    accuracy                           0.78      3862\n",
      "   macro avg       0.68      0.79      0.69      3862\n",
      "weighted avg       0.87      0.78      0.81      3862\n",
      "\n",
      "cohen_kappa_score(CONSERVATIVE_DUMMY): 0.4073040337724173\n",
      "krippendorf(CONSERVATIVE_DUMMY): 0.3847458192745672\n",
      "percent_agreement(CONSERVATIVE_DUMMY): 77.81%\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.94      0.54      0.68      1907\n",
      "           1       0.68      0.96      0.80      1955\n",
      "\n",
      "    accuracy                           0.75      3862\n",
      "   macro avg       0.81      0.75      0.74      3862\n",
      "weighted avg       0.81      0.75      0.74      3862\n",
      "\n",
      "cohen_kappa_score(HAS_OPINION_DUMMY): 0.505378164809953\n",
      "krippendorf(HAS_OPINION_DUMMY): 0.48267588039972054\n",
      "percent_agreement(HAS_OPINION_DUMMY): 75.40%\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.97      0.57      0.72      3165\n",
      "           1       0.32      0.93      0.48       697\n",
      "\n",
      "    accuracy                           0.63      3862\n",
      "   macro avg       0.65      0.75      0.60      3862\n",
      "weighted avg       0.86      0.63      0.68      3862\n",
      "\n",
      "cohen_kappa_score(RATIONALITY_DUMMY): 0.2872630577865316\n",
      "krippendorf(RATIONALITY_DUMMY): 0.19750359705231846\n",
      "percent_agreement(RATIONALITY_DUMMY): 63.49%\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.83      0.69      0.76      1551\n",
      "           1       0.81      0.91      0.86      2311\n",
      "\n",
      "    accuracy                           0.82      3862\n",
      "   macro avg       0.82      0.80      0.81      3862\n",
      "weighted avg       0.82      0.82      0.82      3862\n",
      "\n",
      "cohen_kappa_score(TopicRelevance): 0.6157849447917124\n",
      "krippendorf(TopicRelevance): 0.6138587759369105\n",
      "percent_agreement(TopicRelevance): 82.06%\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.84      0.71      0.77      2856\n",
      "           1       0.43      0.62      0.51      1006\n",
      "\n",
      "    accuracy                           0.68      3862\n",
      "   macro avg       0.63      0.66      0.64      3862\n",
      "weighted avg       0.73      0.68      0.70      3862\n",
      "\n",
      "cohen_kappa_score(INTERACTIVITY_DUMMY): 0.2842644824569087\n",
      "krippendorf(INTERACTIVITY_DUMMY): 0.2726451195670815\n",
      "percent_agreement(INTERACTIVITY_DUMMY): 68.33%\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.90      0.47      0.62      1975\n",
      "           1       0.63      0.94      0.76      1887\n",
      "\n",
      "    accuracy                           0.70      3862\n",
      "   macro avg       0.76      0.71      0.69      3862\n",
      "weighted avg       0.77      0.70      0.69      3862\n",
      "\n",
      "cohen_kappa_score(INCIVILITY_DUMMY): 0.4101926669120459\n",
      "krippendorf(INCIVILITY_DUMMY): 0.37329659527387826\n",
      "percent_agreement(INCIVILITY_DUMMY): 70.20%\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.97      0.70      0.81      3004\n",
      "           1       0.47      0.91      0.62       858\n",
      "\n",
      "    accuracy                           0.75      3862\n",
      "   macro avg       0.72      0.81      0.71      3862\n",
      "weighted avg       0.86      0.75      0.77      3862\n",
      "\n",
      "cohen_kappa_score(HATELIST_FOCUSED_DUMMY): 0.45772876553154684\n",
      "krippendorf(HATELIST_FOCUSED_DUMMY): 0.4296533137087074\n",
      "percent_agreement(HATELIST_FOCUSED_DUMMY): 74.81%\n"
     ]
    }
   ],
   "source": [
    "#these are the new results of the edited prompts for the full dataset:\n",
    "\n",
    "print(\"Inter-coder reliablity: classification_report(human as gold, llm as predicted), cohen_kappa(human, llm):\")\n",
    "\n",
    "for human_col, llm_col in llm_human_column_pairs:\n",
    "    subset = dataset_w_pred_anon[[human_col, llm_col]].dropna()\n",
    "    human = subset[human_col].tolist()\n",
    "    llm = subset[llm_col].tolist()\n",
    "\n",
    "    # Calculate percent agreement\n",
    "    agreement = np.sum(np.array(human) == np.array(llm))/len(np.array(human)) * 100\n",
    "\n",
    "    print(\"---\")\n",
    "    print(f\"{classification_report(human, llm)}\")\n",
    "    print(f\"cohen_kappa_score({human_col}): {cohen_kappa_score(human, llm)}\")\n",
    "    print(f\"krippendorf({human_col}): {krippendorff.alpha(np.array([human, llm]), level_of_measurement=\"nominal\")}\")\n",
    "    print(f\"percent_agreement({human_col}): {agreement:.2f}%\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "id": "94a47574",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>diversity_ideologicaldirection</th>\n",
       "      <th>Absent</th>\n",
       "      <th>Left/Liberal/Democratic</th>\n",
       "      <th>Neutral</th>\n",
       "      <th>Right/Conservative/Republican</th>\n",
       "      <th>Unclear which direction</th>\n",
       "      <th>NaN</th>\n",
       "      <th>All</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Ideologicaldirection</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>58</td>\n",
       "      <td>473</td>\n",
       "      <td>37</td>\n",
       "      <td>196</td>\n",
       "      <td>25</td>\n",
       "      <td>0</td>\n",
       "      <td>789</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1290</td>\n",
       "      <td>230</td>\n",
       "      <td>190</td>\n",
       "      <td>450</td>\n",
       "      <td>72</td>\n",
       "      <td>2</td>\n",
       "      <td>2234</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>11</td>\n",
       "      <td>14</td>\n",
       "      <td>20</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>49</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>30</td>\n",
       "      <td>48</td>\n",
       "      <td>26</td>\n",
       "      <td>83</td>\n",
       "      <td>12</td>\n",
       "      <td>0</td>\n",
       "      <td>199</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>44</td>\n",
       "      <td>29</td>\n",
       "      <td>25</td>\n",
       "      <td>483</td>\n",
       "      <td>8</td>\n",
       "      <td>2</td>\n",
       "      <td>591</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>All</th>\n",
       "      <td>1425</td>\n",
       "      <td>791</td>\n",
       "      <td>292</td>\n",
       "      <td>1232</td>\n",
       "      <td>117</td>\n",
       "      <td>0</td>\n",
       "      <td>3862</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "diversity_ideologicaldirection  Absent  Left/Liberal/Democratic  Neutral  \\\n",
       "Ideologicaldirection                                                       \n",
       "0                                   58                      473       37   \n",
       "1                                 1290                      230      190   \n",
       "2                                    3                       11       14   \n",
       "3                                   30                       48       26   \n",
       "4                                   44                       29       25   \n",
       "All                               1425                      791      292   \n",
       "\n",
       "diversity_ideologicaldirection  Right/Conservative/Republican  \\\n",
       "Ideologicaldirection                                            \n",
       "0                                                         196   \n",
       "1                                                         450   \n",
       "2                                                          20   \n",
       "3                                                          83   \n",
       "4                                                         483   \n",
       "All                                                      1232   \n",
       "\n",
       "diversity_ideologicaldirection  Unclear which direction  NaN   All  \n",
       "Ideologicaldirection                                                \n",
       "0                                                    25    0   789  \n",
       "1                                                    72    2  2234  \n",
       "2                                                     0    1    49  \n",
       "3                                                    12    0   199  \n",
       "4                                                     8    2   591  \n",
       "All                                                 117    0  3862  "
      ]
     },
     "execution_count": 128,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.crosstab(dataset_w_pred_anon.Ideologicaldirection, dataset_w_pred_anon.diversity_ideologicaldirection, margins=True, dropna=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "id": "7a3cff44",
   "metadata": {},
   "outputs": [],
   "source": [
    "#recode the diversity_ideologicaldirection variable:\n",
    "dataset_w_pred_anon.loc[:, 'diversity_ideologicaldirection_recode'] = dataset_w_pred_anon.loc[:, 'diversity_ideologicaldirection'].map({\"Absent\":\"1\", \"Left/Liberal/Democratic\":\"0\", \"Neutral\":\"2\", \"Unclear wich direction\":\"3\", \"Right/Conservative/Republican\":4}).fillna(1).astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "id": "6f620ffb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>diversity_ideologicaldirection_recode</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>4</th>\n",
       "      <th>All</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Ideologicaldirection</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>473</td>\n",
       "      <td>83</td>\n",
       "      <td>37</td>\n",
       "      <td>196</td>\n",
       "      <td>789</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>230</td>\n",
       "      <td>1364</td>\n",
       "      <td>190</td>\n",
       "      <td>450</td>\n",
       "      <td>2234</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>11</td>\n",
       "      <td>4</td>\n",
       "      <td>14</td>\n",
       "      <td>20</td>\n",
       "      <td>49</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>48</td>\n",
       "      <td>42</td>\n",
       "      <td>26</td>\n",
       "      <td>83</td>\n",
       "      <td>199</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>29</td>\n",
       "      <td>54</td>\n",
       "      <td>25</td>\n",
       "      <td>483</td>\n",
       "      <td>591</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>All</th>\n",
       "      <td>791</td>\n",
       "      <td>1547</td>\n",
       "      <td>292</td>\n",
       "      <td>1232</td>\n",
       "      <td>3862</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "diversity_ideologicaldirection_recode    0     1    2     4   All\n",
       "Ideologicaldirection                                             \n",
       "0                                      473    83   37   196   789\n",
       "1                                      230  1364  190   450  2234\n",
       "2                                       11     4   14    20    49\n",
       "3                                       48    42   26    83   199\n",
       "4                                       29    54   25   483   591\n",
       "All                                    791  1547  292  1232  3862"
      ]
     },
     "execution_count": 138,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.crosstab(dataset_w_pred_anon.Ideologicaldirection, dataset_w_pred_anon.diversity_ideologicaldirection_recode, margins=True, dropna=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "id": "9d835314",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Inter-coder reliability: classification_report(human as gold, llm as predicted), cohen_kappa(human, llm):\n",
      "                variable krippendorf percent_agreement\n",
      "0        Acknowledgement        0.27            68.33%\n",
      "1         TopicRelevance        0.61            82.06%\n",
      "2              Reasoning       -0.02            58.31%\n",
      "3         BackgroundInfo        0.34            80.66%\n",
      "4       ExternalEvidence        0.06            79.10%\n",
      "5                Opinion        0.44            72.81%\n",
      "6           disagreement       -0.06            61.76%\n",
      "7   Ideologicaldirection        0.40            60.44%\n",
      "8           Name_calling        0.14            65.77%\n",
      "9              Vulgarity        0.37            87.26%\n",
      "10     Attack_reputation        0.01            50.03%\n",
      "11  Question_Intelligenc        0.01            62.33%\n",
      "12     All_caps_function        0.18            74.37%\n",
      "13  Sarcasm_to_criticize       -0.21            41.66%\n",
      "14      Individual_right        0.02            85.34%\n",
      "15        discrimination        0.05            86.35%\n",
      "16       Invoke_violence        0.20            95.08%\n"
     ]
    }
   ],
   "source": [
    "#evaluate performance against manual coding as if Llama3.1 acted as a human coder:\n",
    "\n",
    "llm_human_column_pairs = [\n",
    "( 'Acknowledgement'\t,\t 'interactivity_acknowledgement_updated_dum'\t),\n",
    "( 'TopicRelevance'\t,\t 'rationality_topic_relevance_dum'\t),\n",
    "( 'Reasoning'\t,\t 'rationality_reasoning_edit_dum'\t),\n",
    "( 'BackgroundInfo'\t,\t 'rationality_background_info_edit_dum'\t),\n",
    "( 'ExternalEvidence'\t,\t 'rationality_external_evidence_edit_dum'\t),\n",
    "( 'Opinion'\t,\t 'diversity_position_dum'\t),\n",
    "( 'disagreement'\t,\t 'diversity_disagreement_dum'\t),\n",
    "( 'Ideologicaldirection'\t,\t 'diversity_ideologicaldirection_recode'\t),\n",
    "( 'Name_calling'\t,\t 'incivility_namecalling_edit_dum'\t),\n",
    "( 'Vulgarity'\t,\t 'incivility_vulgarity_dum'\t),\n",
    "( 'Attack_reputation'\t,\t 'incivility_attackreputation_dum'\t),\n",
    "( 'Question_Intelligenc'\t,\t 'incivility_question_intelligence_dum'\t),\n",
    "( 'All_caps_function'\t,\t'incivility_allcaps_dum'\t),\n",
    "( 'Sarcasm_to_criticize'\t,\t 'incivility_sarcasm_dum'\t),\n",
    "( 'Individual_right'\t,\t 'intolerance_rights_dum'\t),\n",
    "( 'discrimination'\t,\t 'intolerance_discrimination_dum'\t),\n",
    "( 'Invoke_violence'\t,\t 'intolerance_violence_dum'\t)\n",
    "]\n",
    "\n",
    "# Create a list to store the results\n",
    "results = []\n",
    "\n",
    "print(\"Inter-coder reliability: classification_report(human as gold, llm as predicted), cohen_kappa(human, llm):\")\n",
    "\n",
    "for human_col, llm_col in llm_human_column_pairs:\n",
    "    subset = dataset_w_pred_anon[[human_col, llm_col]].dropna()\n",
    "    human = subset[human_col].tolist()\n",
    "    llm = subset[llm_col].tolist()\n",
    "\n",
    "    # Calculate percent agreement\n",
    "    agreement = np.sum(np.array(human) == np.array(llm)) / len(np.array(human)) * 100\n",
    "\n",
    "    # Calculate Krippendorff's alpha\n",
    "    kripp_alpha = krippendorff.alpha(np.array([human, llm]), level_of_measurement=\"nominal\")\n",
    "\n",
    "    # Append the results to the list\n",
    "    results.append({\n",
    "        'variable': human_col,\n",
    "        'krippendorf': f\"{kripp_alpha:.2f}\",\n",
    "        'percent_agreement': f\"{agreement:.2f}%\"\n",
    "    })\n",
    "\n",
    "# Create a DataFrame from the results\n",
    "results_df = pd.DataFrame(results)\n",
    "\n",
    "# Display the DataFrame as a table\n",
    "print(results_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "id": "853bf040",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Results have been written to inter_coder_reliability.csv\n"
     ]
    }
   ],
   "source": [
    "import csv\n",
    "import numpy as np\n",
    "from sklearn.metrics import classification_report, cohen_kappa_score\n",
    "import krippendorff\n",
    "\n",
    "# Assuming llm_human_column_pairs and dataset_w_pred_anon are already defined\n",
    "\n",
    "# Define the CSV file path\n",
    "csv_file_path = 'inter_coder_reliability.csv'\n",
    "\n",
    "# Create a list to store the results\n",
    "results = []\n",
    "\n",
    "# Calculate inter-coder reliability metrics\n",
    "for human_col, llm_col in llm_human_column_pairs:\n",
    "    subset = dataset_w_pred_anon[[human_col, llm_col]].dropna()\n",
    "    human = subset[human_col].tolist()\n",
    "    llm = subset[llm_col].tolist()\n",
    "    \n",
    "    classification_rep = classification_report(human, llm, output_dict=True)\n",
    "    cohen_kappa = cohen_kappa_score(human, llm)\n",
    "    krippendorf_alpha = krippendorff.alpha(np.array([human, llm]), level_of_measurement=\"nominal\")\n",
    "    \n",
    "    # Flatten the classification report dictionary\n",
    "    for label, metrics in classification_rep.items():\n",
    "        if isinstance(metrics, dict):\n",
    "            for metric, value in metrics.items():\n",
    "                results.append({\n",
    "                    'human_col': human_col,\n",
    "                    'llm_col': llm_col,\n",
    "                    'label': label,\n",
    "                    'metric': metric,\n",
    "                    'value': value,\n",
    "                    'cohen_kappa_score': cohen_kappa,\n",
    "                    'krippendorf_alpha': krippendorf_alpha\n",
    "                })\n",
    "        else:\n",
    "            results.append({\n",
    "                'human_col': human_col,\n",
    "                'llm_col': llm_col,\n",
    "                'label': 'overall',\n",
    "                'metric': label,\n",
    "                'value': metrics,\n",
    "                'cohen_kappa_score': cohen_kappa,\n",
    "                'krippendorf_alpha': krippendorf_alpha\n",
    "            })\n",
    "\n",
    "# Write the results to a CSV file\n",
    "with open(csv_file_path, mode='w', newline='') as csv_file:\n",
    "    fieldnames = ['human_col', 'llm_col', 'label', 'metric', 'value', 'cohen_kappa_score', 'krippendorf_alpha']\n",
    "    writer = csv.DictWriter(csv_file, fieldnames=fieldnames)\n",
    "    \n",
    "    writer.writeheader()\n",
    "    for result in results:\n",
    "        writer.writerow(result)\n",
    "\n",
    "print(f\"Results have been written to {csv_file_path}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "3454db04",
   "metadata": {},
   "outputs": [],
   "source": [
    "#check whether lower performance of diversity_conservative__dum is due to token restrictions in options:\n",
    "options_str = \"\"\"\n",
    "seed: 42\n",
    "temperature: 0.1\n",
    "num_predict: 300\n",
    "\"\"\"\n",
    "#note default = num_predict: 128\n",
    "\n",
    "options_long = yaml.safe_load(options_str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "132174d6",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "classifying diversity_ideologicaldirection: 100%|██████████| 3862/3862 [29:22<00:00,  2.19it/s]  \n"
     ]
    }
   ],
   "source": [
    "pubspherepromptsrun17 = ['diversity_ideological_direction']   \n",
    "predictions17: typing.Dict[str, np.ndarray] = {\n",
    "    label: (\n",
    "        src.PromptClassify\n",
    "        .from_json(path)\n",
    "        (dataset[\"commentText\"], model=MODELlarge, options=options_long)\n",
    "    )\n",
    "    for label, path in CFG.prompt_classify_files.items() if label in pubspherepromptsrun17\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "a78ee478",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'diversity_ideological_direction': 0                              Absent\n",
       " 1             Left/Liberal/Democratic\n",
       " 2             Left/Liberal/Democratic\n",
       " 3                             Neutral\n",
       " 4                              Absent\n",
       "                     ...              \n",
       " 3857          Left/Liberal/Democratic\n",
       " 3858    Right/Conservative/Republican\n",
       " 3859                           Absent\n",
       " 3860                           Absent\n",
       " 3861                          Neutral\n",
       " Name: diversity_ideologicaldirection, Length: 3862, dtype: object}"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictions17"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "id": "f81bbaff",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['StartDate',\n",
       " 'RecordedDate',\n",
       " 'IPAddress',\n",
       " 'Finished',\n",
       " 'Coder',\n",
       " 'ID',\n",
       " 'Mark_ID',\n",
       " 'Genre',\n",
       " 'topiccode',\n",
       " 'Platform',\n",
       " 'Anonymity',\n",
       " 'Anonymity_9_TEXT',\n",
       " 'codable',\n",
       " 'Interaction',\n",
       " 'Acknowledgement',\n",
       " 'TopicRelevance',\n",
       " 'Reasoning',\n",
       " 'BackgroundInfo',\n",
       " 'ExternalEvidence',\n",
       " 'ExternalEvidence_1_TEXT',\n",
       " 'Opinion',\n",
       " 'disagreement',\n",
       " 'Ideologicaldirection',\n",
       " 'Name_calling',\n",
       " 'Vulgarity',\n",
       " 'Attack_reputation',\n",
       " 'Question_Intelligenc',\n",
       " 'All_caps_function',\n",
       " 'Sarcasm_to_criticize',\n",
       " 'Individual_right',\n",
       " 'discrimination',\n",
       " 'Invoke_violence',\n",
       " 'Tone',\n",
       " 'INTERACTIVITY_DUMMY',\n",
       " 'RATIONALITY_DUMMY',\n",
       " 'HAS_OPINION_DUMMY',\n",
       " 'LIBERAL_NEUTRAL_CONSERVATIVE',\n",
       " 'LIBERAL_DUMMY',\n",
       " 'CONSERVATIVE_DUMMY',\n",
       " 'NAMECALLING_DUMMY',\n",
       " 'VULGAR_DUMMY',\n",
       " 'NAMECALLING_VULGAR_DUMMY',\n",
       " 'INCIVILITY_ORDINAL',\n",
       " 'INCIVILITY_DUMMY',\n",
       " 'INTOLERANCE_DUMMY',\n",
       " 'filter_$',\n",
       " 'IMPOLITENESS_DUMMY',\n",
       " 'showName',\n",
       " 'genre',\n",
       " 'Time_comment',\n",
       " 'likeCount_comment',\n",
       " 'entities',\n",
       " 'place',\n",
       " 'retweet_count',\n",
       " 'platform',\n",
       " 'retweeted',\n",
       " 'language',\n",
       " 'source',\n",
       " 'in_reply_to_status_id_str',\n",
       " 'in_reply_to_user_id_str',\n",
       " 'in_reply_to_screen_name',\n",
       " 'is_quote_status',\n",
       " 'videoTitle',\n",
       " 'description',\n",
       " 'Time_video',\n",
       " 'channelTitle',\n",
       " 'channelId',\n",
       " 'viewCount',\n",
       " 'dislikeCount_video',\n",
       " 'likeCount_video',\n",
       " 'date_difference',\n",
       " 'commentCount_video',\n",
       " 'replyCount_comment',\n",
       " 'topic',\n",
       " 'subscribers',\n",
       " 'HATELIST_FOCUSED_DUMMY',\n",
       " 'Time_comment_year',\n",
       " 'Time_video_year',\n",
       " 'interactivity_acknowledgement',\n",
       " 'rationality_external_evidence',\n",
       " 'rationality_topic_relevance',\n",
       " 'political_negativity',\n",
       " 'rationality_background_info',\n",
       " 'rationality_reasoning',\n",
       " 'topics',\n",
       " 'emotions',\n",
       " 'sentiment',\n",
       " 'irony',\n",
       " 'offensive',\n",
       " 'hate',\n",
       " 'commentText',\n",
       " 'political_ideology_US',\n",
       " 'diversity_disagreement',\n",
       " 'diversity_ideologicaldirection',\n",
       " 'diversity_position',\n",
       " 'incivility_allcaps',\n",
       " 'incivility_attackreputation',\n",
       " 'incivility_namecalling',\n",
       " 'incivility_question_intelligence',\n",
       " 'incivility_sarcasm',\n",
       " 'incivility_vulgarity',\n",
       " 'diversity_disagreement_dum',\n",
       " 'diversity_position_dum',\n",
       " 'incivility_allcaps_dum',\n",
       " 'incivility_attackreputation_dum',\n",
       " 'incivility_question_intelligence_dum',\n",
       " 'incivility_sarcasm_dum',\n",
       " 'incivility_namecalling_dum',\n",
       " 'incivility_vulgarity_dum',\n",
       " 'diversity_ideologicaldirection_dum',\n",
       " 'interactivity_acknowledgement_low',\n",
       " 'intolerance_discrimination',\n",
       " 'intolerance_rights',\n",
       " 'intolerance_violence',\n",
       " 'political_ideology_US_low',\n",
       " 'rationality_background_info_low',\n",
       " 'rationality_external_evidence_low',\n",
       " 'rationality_reasoning_low',\n",
       " 'rationality_topic_relevance_low',\n",
       " 'incivility_namecalling_low2',\n",
       " 'rationality_background_info_updated',\n",
       " 'rationality_external_evidence_updated',\n",
       " 'rationality_background_info_run2',\n",
       " 'rationality_external_evidence_run2',\n",
       " 'rationality_external_evidence_edit',\n",
       " 'rationality_background_info_edit',\n",
       " 'interactivity_acknowledgement_updated',\n",
       " 'rationality_reasoning_edit',\n",
       " 'incivility_namecalling_updated',\n",
       " 'incivility_namecalling_edit',\n",
       " 'diversity_ideologicaldirection_long']"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset_w_pred_2.columns.to_list()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "e0ceee6e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "diversity_ideologicaldirection\n",
      "Absent                           1454\n",
      "Right/Conservative/Republican    1165\n",
      "Left/Liberal/Democratic           805\n",
      "Neutral                           320\n",
      "Unclear which direction           113\n",
      "Name: count, dtype: int64\n",
      "------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "#join to the dataset:   \n",
    "for _, preds in predictions17.items():\n",
    "    print(preds.value_counts())\n",
    "    print(\"-\" * 42)\n",
    "    preds_df = pd.DataFrame(preds).add_suffix('_long')\n",
    "   \n",
    "    dataset_w_pred_2 = dataset_w_pred_2.join(preds_df)\n",
    "    dataset_w_pred_anon = dataset_w_pred_anon.join(preds_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "id": "ebb350bf",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_w_pred_2.to_json(f'{CFG.report_dir}/publicsphere.cardiff_prompt_classify_s.json', orient=\"records\", force_ascii=False, indent=4)\n",
    "dataset_w_pred_2.to_parquet(f'{CFG.report_dir}/publicsphere.cardiff_prompt_classify_s.parquet')\n",
    "dataset_w_pred_anon.to_json('data/publicsphere/publicsphere.cardiff_prompt_classify_anon.json', orient=\"records\", force_ascii=False, indent=4)\n",
    "dataset_w_pred_anon.to_parquet('data/publicsphere/publicsphere.cardiff_prompt_classify_anon.parquet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "id": "4eb7de0c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "diversity_ideologicaldirection_long  Absent  Left/Liberal/Democratic  Neutral  \\\n",
      "diversity_ideologicaldirection                                                  \n",
      "Absent                                 1399                        6       10   \n",
      "Left/Liberal/Democratic                   5                      759        9   \n",
      "Neutral                                  10                       12      255   \n",
      "Right/Conservative/Republican            26                       28       44   \n",
      "Unclear which direction                  14                        0        1   \n",
      "NaN                                       0                        0        1   \n",
      "All                                    1454                      805      320   \n",
      "\n",
      "diversity_ideologicaldirection_long  Right/Conservative/Republican  \\\n",
      "diversity_ideologicaldirection                                       \n",
      "Absent                                                           7   \n",
      "Left/Liberal/Democratic                                         18   \n",
      "Neutral                                                         12   \n",
      "Right/Conservative/Republican                                 1127   \n",
      "Unclear which direction                                          1   \n",
      "NaN                                                              0   \n",
      "All                                                           1165   \n",
      "\n",
      "diversity_ideologicaldirection_long  Unclear which direction  NaN     All  \n",
      "diversity_ideologicaldirection                                             \n",
      "Absent                                                     3    0  1425.0  \n",
      "Left/Liberal/Democratic                                    0    0   791.0  \n",
      "Neutral                                                    2    1   292.0  \n",
      "Right/Conservative/Republican                              7    0  1232.0  \n",
      "Unclear which direction                                  101    0   117.0  \n",
      "NaN                                                        0    4     NaN  \n",
      "All                                                      113    0  3862.0  \n"
     ]
    }
   ],
   "source": [
    "#compare run-rerun results:\n",
    "print(pd.crosstab(dataset_w_pred_anon['diversity_ideologicaldirection'], dataset_w_pred_anon['diversity_ideologicaldirection_long'], margins=True, dropna=False))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "id": "026d9403",
   "metadata": {},
   "outputs": [],
   "source": [
    "#recode into dummies:\n",
    "dataset_w_pred_anon.loc[:, 'diversity_conservative_long_dum'] = dataset_w_pred_anon.loc[:, 'diversity_ideologicaldirection_long'].map({\"Left/Liberal/Democratic\": 0, \"Absent\":0, \"Neutral\":0, \"Unclear which direction\":0 , \"Right/Conservative/Republican\":1}).fillna(0).astype(int)\n",
    "dataset_w_pred_anon.loc[:, 'diversity_liberal_long_dum'] = dataset_w_pred_anon.loc[:, 'diversity_ideologicaldirection_long'].map({\"Left/Liberal/Democratic\": 1, \"Absent\":0, \"Neutral\":0, \"Unclear which direction\":0 , \"Right/Conservative/Republican\":0}).fillna(0).astype(int)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "id": "aa285fff",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Inter-coder reliablity: classification_report(human as gold, llm as predicted), cohen_kappa(human, llm):\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.90      0.89      0.89      3073\n",
      "           1       0.59      0.60      0.60       789\n",
      "\n",
      "    accuracy                           0.83      3862\n",
      "   macro avg       0.74      0.75      0.75      3862\n",
      "weighted avg       0.83      0.83      0.83      3862\n",
      "\n",
      "cohen_kappa_score(LIBERAL_DUMMY): 0.4909411843205673\n",
      "krippendorf(LIBERAL_DUMMY): 0.490993755129861\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.96      0.79      0.86      3271\n",
      "           1       0.41      0.80      0.54       591\n",
      "\n",
      "    accuracy                           0.79      3862\n",
      "   macro avg       0.68      0.79      0.70      3862\n",
      "weighted avg       0.87      0.79      0.81      3862\n",
      "\n",
      "cohen_kappa_score(CONSERVATIVE_DUMMY): 0.42119755573079587\n",
      "krippendorf(CONSERVATIVE_DUMMY): 0.40307780447886066\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.90      0.90      0.90      3073\n",
      "           1       0.60      0.60      0.60       789\n",
      "\n",
      "    accuracy                           0.84      3862\n",
      "   macro avg       0.75      0.75      0.75      3862\n",
      "weighted avg       0.84      0.84      0.84      3862\n",
      "\n",
      "cohen_kappa_score(LIBERAL_DUMMY): 0.49554428091035374\n",
      "krippendorf(LIBERAL_DUMMY): 0.49560938324103376\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.96      0.77      0.85      3271\n",
      "           1       0.39      0.82      0.53       591\n",
      "\n",
      "    accuracy                           0.78      3862\n",
      "   macro avg       0.68      0.79      0.69      3862\n",
      "weighted avg       0.87      0.78      0.81      3862\n",
      "\n",
      "cohen_kappa_score(CONSERVATIVE_DUMMY): 0.4073040337724173\n",
      "krippendorf(CONSERVATIVE_DUMMY): 0.3847458192745672\n"
     ]
    }
   ],
   "source": [
    "#evaluete respective performance:\n",
    "llm_human_column_pairs = [\n",
    "    (\"LIBERAL_DUMMY\", \"diversity_liberal_long_dum\"),\n",
    "    (\"CONSERVATIVE_DUMMY\", \"diversity_conservative_long_dum\"),\n",
    "    (\"LIBERAL_DUMMY\", \"diversity_liberal_dum\"),\n",
    "    (\"CONSERVATIVE_DUMMY\", \"diversity_conservative_dum\")\n",
    "]\n",
    "print(\"Inter-coder reliablity: classification_report(human as gold, llm as predicted), cohen_kappa(human, llm):\")\n",
    "\n",
    "for human_col, llm_col in llm_human_column_pairs:\n",
    "    subset = dataset_w_pred_anon[[human_col, llm_col]].dropna()\n",
    "    human = subset[human_col].tolist()\n",
    "    llm = subset[llm_col].tolist()\n",
    "    \n",
    "    print(\"---\")\n",
    "    print(f\"{classification_report(human, llm)}\")\n",
    "    print(f\"cohen_kappa_score({human_col}): {cohen_kappa_score(human, llm)}\")\n",
    "    print(f\"krippendorf({human_col}): {krippendorff.alpha(np.array([human, llm]), level_of_measurement=\"nominal\")}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "aa5f29c0",
   "metadata": {},
   "source": [
    "very similar performance regardless of num_predict token limit settings, makes sense since this limit is about max output tokens in the response, not the prompt, so here is also another test-retest result for the same prompt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5614036d",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_w_pred_2.to_json(f'{CFG.report_dir}/publicsphere.cardiff_prompt_classify_s.json', orient=\"records\", force_ascii=False, indent=4)\n",
    "dataset_w_pred_2.to_parquet(f'{CFG.report_dir}/publicsphere.cardiff_prompt_classify_s.parquet')\n",
    "dataset_w_pred_anon.to_json('data/publicsphere/publicsphere.cardiff_prompt_classify_anon.json', orient=\"records\", force_ascii=False, indent=4)\n",
    "dataset_w_pred_anon.to_parquet('data/publicsphere/publicsphere.cardiff_prompt_classify_anon.parquet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c5bdea7f",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "classifying rationality_combine: 100%|██████████| 3862/3862 [1:30:14<00:00,  1.40s/it]\n"
     ]
    }
   ],
   "source": [
    "#what if we combine rationality items into one prompt:\n",
    "pubspherepromptsrun18 = ['rationality_combine']   \n",
    "predictions18: typing.Dict[str, np.ndarray] = {\n",
    "    label: (\n",
    "        src.PromptClassify\n",
    "        .from_json(path)\n",
    "        (dataset[\"commentText\"], model=MODELlarge, options=options_low)\n",
    "    )\n",
    "    for label, path in CFG.prompt_classify_files.items() if label in pubspherepromptsrun18\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9d6158f0",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "classifying incivility_simple: 100%|██████████| 3862/3862 [27:41<00:00,  2.32it/s]\n",
      "classifying rationality_combine: 100%|██████████| 3862/3862 [33:51<00:00,  1.90it/s]\n"
     ]
    }
   ],
   "source": [
    "#what if we combine incivility items into one prompt:\n",
    "pubspherepromptsrun20 = ['incivility_simple', 'rationality_simple']   \n",
    "predictions20: typing.Dict[str, np.ndarray] = {\n",
    "    label: (\n",
    "        src.PromptClassify\n",
    "        .from_json(path)\n",
    "        (dataset[\"commentText\"], model=MODELlarge, options=options_low)\n",
    "    )\n",
    "    for label, path in CFG.prompt_classify_files.items() if label in pubspherepromptsrun20\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9cd86356",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "classifying incivility_simple2: 100%|██████████| 3862/3862 [32:08<00:00,  2.00it/s] \n",
      "classifying interactivity_acknowledgement_simple: 100%|██████████| 3862/3862 [32:57<00:00,  1.95it/s]\n",
      "classifying rationality_simple2: 100%|██████████| 3862/3862 [46:36<00:00,  1.38it/s] \n"
     ]
    }
   ],
   "source": [
    "#what if we combine incivility items into one prompt:\n",
    "pubspherepromptsrun21 = ['interactivity_acknowledgement_simple', 'incivility_simple2', 'rationality_simple2']   \n",
    "predictions21: typing.Dict[str, np.ndarray] = {\n",
    "    label: (\n",
    "        src.PromptClassify\n",
    "        .from_json(path)\n",
    "        (dataset[\"commentText\"], model=MODELlarge, options=options_low)\n",
    "    )\n",
    "    for label, path in CFG.prompt_classify_files.items() if label in pubspherepromptsrun21\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "65db6a31",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "classifying interactivity_acknowledgement_simple: 100%|██████████| 3862/3862 [28:55<00:00,  2.23it/s]   \n"
     ]
    }
   ],
   "source": [
    "#what if we extend the interactivety first concept item (not in codebook) to the simple prompt:\n",
    "pubspherepromptsrun24 = ['interactivity_acknowledgement_simple2']   \n",
    "predictions26: typing.Dict[str, np.ndarray] = {\n",
    "    label: (\n",
    "        src.PromptClassify\n",
    "        .from_json(path)\n",
    "        (dataset[\"commentText\"], model=MODELlarge, options=options_low)\n",
    "    )\n",
    "    for label, path in CFG.prompt_classify_files.items() if label in pubspherepromptsrun24\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "03649939",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "rationality_combine\n",
      "No     2473\n",
      "Yes    1386\n",
      "Name: count, dtype: int64\n",
      "------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "#join to the dataset:   \n",
    "for _, preds in predictions18.items():\n",
    "    print(preds.value_counts())\n",
    "    print(\"-\" * 42)\n",
    "     \n",
    "    dataset_w_pred_2 = dataset_w_pred_2.join(preds, rsuffix='_exactexample')\n",
    "    dataset_w_pred_anon = dataset_w_pred_anon.join(preds, rsuffix='_exactexample')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "ff3b08c4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['StartDate',\n",
       " 'RecordedDate',\n",
       " 'IPAddress',\n",
       " 'Finished',\n",
       " 'Coder',\n",
       " 'ID',\n",
       " 'Mark_ID',\n",
       " 'Genre',\n",
       " 'topiccode',\n",
       " 'Platform',\n",
       " 'Anonymity',\n",
       " 'Anonymity_9_TEXT',\n",
       " 'codable',\n",
       " 'Interaction',\n",
       " 'Acknowledgement',\n",
       " 'TopicRelevance',\n",
       " 'Reasoning',\n",
       " 'BackgroundInfo',\n",
       " 'ExternalEvidence',\n",
       " 'ExternalEvidence_1_TEXT',\n",
       " 'Opinion',\n",
       " 'disagreement',\n",
       " 'Ideologicaldirection',\n",
       " 'Name_calling',\n",
       " 'Vulgarity',\n",
       " 'Attack_reputation',\n",
       " 'Question_Intelligenc',\n",
       " 'All_caps_function',\n",
       " 'Sarcasm_to_criticize',\n",
       " 'Individual_right',\n",
       " 'discrimination',\n",
       " 'Invoke_violence',\n",
       " 'Tone',\n",
       " 'INTERACTIVITY_DUMMY',\n",
       " 'RATIONALITY_DUMMY',\n",
       " 'HAS_OPINION_DUMMY',\n",
       " 'LIBERAL_NEUTRAL_CONSERVATIVE',\n",
       " 'LIBERAL_DUMMY',\n",
       " 'CONSERVATIVE_DUMMY',\n",
       " 'NAMECALLING_DUMMY',\n",
       " 'VULGAR_DUMMY',\n",
       " 'NAMECALLING_VULGAR_DUMMY',\n",
       " 'INCIVILITY_ORDINAL',\n",
       " 'INCIVILITY_DUMMY',\n",
       " 'INTOLERANCE_DUMMY',\n",
       " 'filter_$',\n",
       " 'IMPOLITENESS_DUMMY',\n",
       " 'showName',\n",
       " 'genre',\n",
       " 'Time_comment',\n",
       " 'likeCount_comment',\n",
       " 'entities',\n",
       " 'place',\n",
       " 'retweet_count',\n",
       " 'platform',\n",
       " 'retweeted',\n",
       " 'language',\n",
       " 'source',\n",
       " 'in_reply_to_status_id_str',\n",
       " 'in_reply_to_user_id_str',\n",
       " 'in_reply_to_screen_name',\n",
       " 'is_quote_status',\n",
       " 'videoTitle',\n",
       " 'description',\n",
       " 'Time_video',\n",
       " 'channelTitle',\n",
       " 'channelId',\n",
       " 'viewCount',\n",
       " 'dislikeCount_video',\n",
       " 'likeCount_video',\n",
       " 'date_difference',\n",
       " 'commentCount_video',\n",
       " 'replyCount_comment',\n",
       " 'topic',\n",
       " 'subscribers',\n",
       " 'HATELIST_FOCUSED_DUMMY',\n",
       " 'Time_comment_year',\n",
       " 'Time_video_year',\n",
       " 'interactivity_acknowledgement_old',\n",
       " 'rationality_external_evidence_old',\n",
       " 'rationality_topic_relevance_old',\n",
       " 'political_negativity_old',\n",
       " 'rationality_background_info_old',\n",
       " 'rationality_reasoning_old',\n",
       " 'topics',\n",
       " 'emotions',\n",
       " 'sentiment',\n",
       " 'irony',\n",
       " 'offensive',\n",
       " 'hate',\n",
       " 'political_ideology_US_old',\n",
       " 'offensive_dum',\n",
       " 'hate_dum',\n",
       " 'cardiff_incivil',\n",
       " 'rationality_reasoning_dum_old',\n",
       " 'rationality_background_info_dum_old',\n",
       " 'rationality_external_evidence_dum_old',\n",
       " 'rationality_prompt_dum_old',\n",
       " 'political_conservative_US_old',\n",
       " 'political_liberal_US_old',\n",
       " 'political_opinion_US_old',\n",
       " 'diversity_positiondum_old',\n",
       " 'diversity_disagreement',\n",
       " 'diversity_ideologicaldirection',\n",
       " 'diversity_position',\n",
       " 'incivility_allcaps',\n",
       " 'incivility_attackreputation',\n",
       " 'incivility_question_intelligence',\n",
       " 'incivility_sarcasm',\n",
       " 'incivility_namecalling_NA',\n",
       " 'incivility_vulgarity',\n",
       " 'diversity_positiondum_dum_old',\n",
       " 'diversity_disagreement_dum',\n",
       " 'diversity_position_dum',\n",
       " 'incivility_allcaps_dum',\n",
       " 'incivility_attackreputation_dum',\n",
       " 'incivility_question_intelligence_dum',\n",
       " 'incivility_sarcasm_dum',\n",
       " 'incivility_namecalling_dum_NA',\n",
       " 'incivility_vulgarity_dum',\n",
       " 'diversity_ideologicaldirection_dum',\n",
       " 'interactivity_acknowledgement_low',\n",
       " 'intolerance_discrimination',\n",
       " 'intolerance_rights',\n",
       " 'intolerance_violence',\n",
       " 'political_ideology_US_low',\n",
       " 'rationality_background_info_low',\n",
       " 'rationality_external_evidence_low',\n",
       " 'rationality_reasoning_low',\n",
       " 'rationality_topic_relevance_low',\n",
       " 'incivility_namecalling',\n",
       " 'rationality_background_info_updated',\n",
       " 'rationality_external_evidence_updated',\n",
       " 'interactivity_acknowledgement_dum',\n",
       " 'rationality_external_evidence_dum',\n",
       " 'rationality_topic_relevance_dum',\n",
       " 'rationality_background_info_low_dum',\n",
       " 'rationality_reasoning_dum',\n",
       " 'intolerance_discrimination_dum',\n",
       " 'intolerance_rights_dum',\n",
       " 'intolerance_violence_dum',\n",
       " 'incivility_namecalling_dum',\n",
       " 'political_conservative_US_dum',\n",
       " 'political_liberal_US_dum',\n",
       " 'political_opinion_US_dum',\n",
       " 'diversity_liberal_dum',\n",
       " 'rationality_background_info_updated_dum',\n",
       " 'rationality_external_evidence_updated_dum',\n",
       " 'rationality_prompt_dum',\n",
       " 'incivility_prompt_dum',\n",
       " 'hate_list_prompt_dum',\n",
       " 'rationality_background_info_run2',\n",
       " 'rationality_external_evidence_run2',\n",
       " 'rationality_external_evidence_edit',\n",
       " 'rationality_background_info_edit',\n",
       " 'interactivity_acknowledgement_updated',\n",
       " 'rationality_reasoning_edit',\n",
       " 'incivility_namecalling_updated',\n",
       " 'incivility_namecalling_edit',\n",
       " 'diversity_conservative_dum',\n",
       " 'interactivity_acknowledgement_updated_dum',\n",
       " 'incivility_namecalling_updated_dum',\n",
       " 'incivility_namecalling_edit_dum',\n",
       " 'rationality_reasoning_edit_dum',\n",
       " 'rationality_background_info_run2_dum',\n",
       " 'rationality_external_evidence_run2_dum',\n",
       " 'rationality_external_evidence_edit_dum',\n",
       " 'rationality_background_info_edit_dum',\n",
       " 'diversity_ideologicaldirection_long',\n",
       " 'diversity_conservative_long_dum',\n",
       " 'diversity_liberal_long_dum',\n",
       " 'diversity_disagreement_edit',\n",
       " 'diversity_disagreement_edit_dum',\n",
       " 'rationality_combine',\n",
       " 'rationality_combine_dum',\n",
       " 'incivility_combined',\n",
       " 'incivility_combine_dum',\n",
       " 'rationality_combine_exactexample',\n",
       " 'incivility_simple',\n",
       " 'rationality_combine_simple']"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset_w_pred_anon.columns.to_list()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "833abd73",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "rationality_combine                 No   Yes  NaN     All\n",
      "rationality_combine_exactexample                         \n",
      "No                                2394    79    0  2473.0\n",
      "Yes                                 48  1338    0  1386.0\n",
      "NaN                                  1     0    2     NaN\n",
      "All                               2443  1417    0  3862.0\n"
     ]
    }
   ],
   "source": [
    "#compare influence of exact versus similar example:\n",
    "print(pd.crosstab(dataset_w_pred_anon['rationality_combine_exactexample'], dataset_w_pred_anon['rationality_combine'], margins=True, dropna=False))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "182a7563",
   "metadata": {},
   "source": [
    "very small effect"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "2ebfa66b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "incivility_simple2\n",
      "Yes    2414\n",
      "No     1448\n",
      "Name: count, dtype: int64\n",
      "------------------------------------------\n",
      "interactivity_acknowledgement_simple\n",
      "No     2318\n",
      "Yes    1544\n",
      "Name: count, dtype: int64\n",
      "------------------------------------------\n",
      "rationality_simple2\n",
      "No     3489\n",
      "Yes     373\n",
      "Name: count, dtype: int64\n",
      "------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "#join to the dataset:   \n",
    "for _, preds in predictions21.items():\n",
    "    print(preds.value_counts())\n",
    "    print(\"-\" * 42)\n",
    "     \n",
    "    dataset_w_pred_2 = dataset_w_pred_2.join(preds, rsuffix='_2')\n",
    "    dataset_w_pred_anon = dataset_w_pred_anon.join(preds, rsuffix='_2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "828b97b9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "interactivity_acknowledgement_simple\n",
      "No     1954\n",
      "Yes    1907\n",
      "Name: count, dtype: int64\n",
      "------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "#join to the dataset:   \n",
    "for _, preds in predictions26.items():\n",
    "    print(preds.value_counts())\n",
    "    print(\"-\" * 42)\n",
    "     \n",
    "    dataset_w_pred_2 = dataset_w_pred_2.join(preds, rsuffix='_2')\n",
    "    dataset_w_pred_anon = dataset_w_pred_anon.join(preds, rsuffix='_2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "69ca3b5d",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_w_pred_anon.loc[:, 'rationality_combine_exactexample_dum'] = dataset_w_pred_2.loc[:, 'rationality_combine_exactexample'].map({\"Yes\": 1, \"No\":0}).fillna(0).astype(int)\n",
    "dataset_w_pred_anon.loc[:, 'rationality_combine_dum'] = dataset_w_pred_2.loc[:, 'rationality_combine'].map({\"Yes\": 1, \"No\":0}).fillna(0).astype(int)\n",
    "dataset_w_pred_anon.loc[:, 'rationality_simple_dum'] = dataset_w_pred_2.loc[:, 'rationality_simple'].map({\"Yes\": 1, \"No\":0}).fillna(0).astype(int)\n",
    "dataset_w_pred_anon.loc[:, 'incivility_combine_dum'] = dataset_w_pred_2.loc[:, 'incivility_combined'].map({\"Yes\": 1, \"No\":0}).fillna(0).astype(int)\n",
    "dataset_w_pred_anon.loc[:, 'incivility_simple_dum'] = dataset_w_pred_2.loc[:, 'incivility_simple'].map({\"Yes\": 1, \"No\":0}).fillna(0).astype(int)\n",
    "dataset_w_pred_anon.loc[:, 'incivility_simple2_dum'] = dataset_w_pred_2.loc[:, 'incivility_simple2'].map({\"Yes\": 1, \"No\":0}).fillna(0).astype(int)\n",
    "dataset_w_pred_anon.loc[:, 'rationality_simple2_dum'] = dataset_w_pred_2.loc[:, 'rationality_simple2'].map({\"Yes\": 1, \"No\":0}).fillna(0).astype(int)\n",
    "dataset_w_pred_anon.loc[:, 'interactivity_acknowledgement_simple_dum'] = dataset_w_pred_2.loc[:, 'interactivity_acknowledgement_simple'].map({\"Yes\": 1, \"No\":0}).fillna(0).astype(int)\n",
    "dataset_w_pred_anon.loc[:, 'interactivity_acknowledgement_simple2_dum'] = dataset_w_pred_2.loc[:, 'interactivity_acknowledgement_simple_2'].map({\"Yes\": 1, \"No\":0}).fillna(0).astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "4bc781dc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>in_reply_to_status_id_str</th>\n",
       "      <th>in_reply_to_user_id_str</th>\n",
       "      <th>in_reply_to_screen_name</th>\n",
       "      <th>interactivity_acknowledgement_simple2_dum</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.161857e+18</td>\n",
       "      <td>7.713886e+17</td>\n",
       "      <td>11thHour</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   in_reply_to_status_id_str  in_reply_to_user_id_str in_reply_to_screen_name  \\\n",
       "0                        NaN                      NaN                    None   \n",
       "1                        NaN                      NaN                    None   \n",
       "2               1.161857e+18             7.713886e+17                11thHour   \n",
       "3                        NaN                      NaN                    None   \n",
       "4                        NaN                      NaN                    None   \n",
       "\n",
       "   interactivity_acknowledgement_simple2_dum  \n",
       "0                                          0  \n",
       "1                                          0  \n",
       "2                                          1  \n",
       "3                                          0  \n",
       "4                                          0  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset_w_pred_anon.loc[:, [ 'in_reply_to_status_id_str', 'in_reply_to_user_id_str', 'in_reply_to_screen_name', 'interactivity_acknowledgement_simple2_dum']].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "387b540d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\sstolwi\\AppData\\Local\\Temp\\ipykernel_22972\\593970346.py:2: FutureWarning: DataFrame.applymap has been deprecated. Use DataFrame.map instead.\n",
      "  dataset_w_pred_anon.loc[:, 'is_reply_sum'] = dataset_w_pred_anon.loc[:, ['in_reply_to_status_id_str', 'in_reply_to_user_id_str', 'in_reply_to_screen_name']].applymap(lambda x: 1 if pd.notna(x) and x is not None else 0).max(axis=1)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>in_reply_to_status_id_str</th>\n",
       "      <th>in_reply_to_user_id_str</th>\n",
       "      <th>in_reply_to_screen_name</th>\n",
       "      <th>interactivity_acknowledgement_simple_dum</th>\n",
       "      <th>is_reply_sum</th>\n",
       "      <th>interactivity_tot_simple_dum</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.161857e+18</td>\n",
       "      <td>7.713886e+17</td>\n",
       "      <td>11thHour</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   in_reply_to_status_id_str  in_reply_to_user_id_str in_reply_to_screen_name  \\\n",
       "0                        NaN                      NaN                    None   \n",
       "1                        NaN                      NaN                    None   \n",
       "2               1.161857e+18             7.713886e+17                11thHour   \n",
       "3                        NaN                      NaN                    None   \n",
       "4                        NaN                      NaN                    None   \n",
       "\n",
       "   interactivity_acknowledgement_simple_dum  is_reply_sum  \\\n",
       "0                                         0             0   \n",
       "1                                         0             0   \n",
       "2                                         1             1   \n",
       "3                                         0             0   \n",
       "4                                         0             0   \n",
       "\n",
       "   interactivity_tot_simple_dum  \n",
       "0                             0  \n",
       "1                             0  \n",
       "2                             1  \n",
       "3                             0  \n",
       "4                             0  "
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# create a dummy variable for the interactivity concept, which is 1 if either one of ['in_reply_to_status_id_str', 'in_reply_to_user_id_str', 'in_reply_to_screen_name'] is not NA, or interactivity_acknowledgement_simple is 1:\n",
    "dataset_w_pred_anon.loc[:, 'is_reply_sum'] = dataset_w_pred_anon.loc[:, ['in_reply_to_status_id_str', 'in_reply_to_user_id_str', 'in_reply_to_screen_name']].applymap(lambda x: 1 if pd.notna(x) and x is not None else 0).max(axis=1)\n",
    "dataset_w_pred_anon.loc[:, 'interactivity_tot_simple_dum'] = dataset_w_pred_anon.loc[:, ['is_reply_sum', 'interactivity_acknowledgement_simple_dum']].max(axis=1).astype(int)\n",
    "\n",
    "dataset_w_pred_anon.loc[:, [ 'in_reply_to_status_id_str', 'in_reply_to_user_id_str', 'in_reply_to_screen_name', 'interactivity_acknowledgement_simple_dum', 'is_reply_sum', 'interactivity_tot_simple_dum']].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "3246b9f8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>interactivity_acknowledgement_simple2_dum</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>All</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>interactivity_tot_simple2_dum</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1836</td>\n",
       "      <td>0</td>\n",
       "      <td>1836</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>119</td>\n",
       "      <td>1907</td>\n",
       "      <td>2026</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>All</th>\n",
       "      <td>1955</td>\n",
       "      <td>1907</td>\n",
       "      <td>3862</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "interactivity_acknowledgement_simple2_dum     0     1   All\n",
       "interactivity_tot_simple2_dum                              \n",
       "0                                          1836     0  1836\n",
       "1                                           119  1907  2026\n",
       "All                                        1955  1907  3862"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.crosstab(dataset_w_pred_anon.interactivity_tot_simple2_dum, dataset_w_pred_anon.interactivity_acknowledgement_simple2_dum, margins=True, dropna=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "9bf8ef00",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>INTERACTIVITY_DUMMY</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>All</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>is_reply_sum</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2157</td>\n",
       "      <td>975</td>\n",
       "      <td>3132</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>699</td>\n",
       "      <td>31</td>\n",
       "      <td>730</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>All</th>\n",
       "      <td>2856</td>\n",
       "      <td>1006</td>\n",
       "      <td>3862</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "INTERACTIVITY_DUMMY     0     1   All\n",
       "is_reply_sum                         \n",
       "0                    2157   975  3132\n",
       "1                     699    31   730\n",
       "All                  2856  1006  3862"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.crosstab(dataset_w_pred_anon.is_reply_sum, dataset_w_pred_anon.INTERACTIVITY_DUMMY, margins=True, dropna=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "753fd9b7",
   "metadata": {},
   "outputs": [],
   "source": [
    "#the interactivity manual coded dummy does not always turn one if the comment is a reply, in fact it is mostly zero, so this can't be the way the variable is operationalized."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "278de7a5",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_w_pred_2.to_json(f'{CFG.report_dir}/publicsphere.cardiff_prompt_classify_s.json', orient=\"records\", force_ascii=False, indent=4)\n",
    "dataset_w_pred_2.to_parquet(f'{CFG.report_dir}/publicsphere.cardiff_prompt_classify_s.parquet')\n",
    "dataset_w_pred_anon.to_json('data/publicsphere/publicsphere.cardiff_prompt_classify_anon.json', orient=\"records\", force_ascii=False, indent=4)\n",
    "dataset_w_pred_anon.to_parquet('data/publicsphere/publicsphere.cardiff_prompt_classify_anon.parquet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "73804e6e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "16f6123d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Inter-coder reliablity: classification_report(human as gold, llm as predicted), cohen_kappa(human, llm):\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.93      0.72      0.81      2541\n",
      "           1       0.37      0.76      0.50       548\n",
      "\n",
      "    accuracy                           0.73      3089\n",
      "   macro avg       0.65      0.74      0.66      3089\n",
      "weighted avg       0.83      0.73      0.76      3089\n",
      "\n",
      "cohen_kappa_score(RATIONALITY_DUMMY): 0.34450253716933377\n",
      "krippendorf(RATIONALITY_DUMMY): 0.315657770173673\n",
      "percent_agreement(RATIONALITY_DUMMY): 72.97%\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.94      0.73      0.82      2541\n",
      "           1       0.38      0.77      0.51       548\n",
      "\n",
      "    accuracy                           0.74      3089\n",
      "   macro avg       0.66      0.75      0.66      3089\n",
      "weighted avg       0.84      0.74      0.77      3089\n",
      "\n",
      "cohen_kappa_score(RATIONALITY_DUMMY): 0.35616906588990993\n",
      "krippendorf(RATIONALITY_DUMMY): 0.32974372621529047\n",
      "percent_agreement(RATIONALITY_DUMMY): 73.75%\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.88      0.88      0.88      2541\n",
      "           1       0.45      0.45      0.45       548\n",
      "\n",
      "    accuracy                           0.81      3089\n",
      "   macro avg       0.67      0.67      0.67      3089\n",
      "weighted avg       0.81      0.81      0.81      3089\n",
      "\n",
      "cohen_kappa_score(RATIONALITY_DUMMY): 0.3341706978553465\n",
      "krippendorf(RATIONALITY_DUMMY): 0.3342773940900393\n",
      "percent_agreement(RATIONALITY_DUMMY): 80.61%\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.87      0.96      0.92      2541\n",
      "           1       0.66      0.36      0.46       548\n",
      "\n",
      "    accuracy                           0.85      3089\n",
      "   macro avg       0.77      0.66      0.69      3089\n",
      "weighted avg       0.84      0.85      0.84      3089\n",
      "\n",
      "cohen_kappa_score(RATIONALITY_DUMMY): 0.38727369911274523\n",
      "krippendorf(RATIONALITY_DUMMY): 0.37857585459051946\n",
      "percent_agreement(RATIONALITY_DUMMY): 85.37%\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.78      0.75      0.77      1567\n",
      "           1       0.75      0.79      0.77      1522\n",
      "\n",
      "    accuracy                           0.77      3089\n",
      "   macro avg       0.77      0.77      0.77      3089\n",
      "weighted avg       0.77      0.77      0.77      3089\n",
      "\n",
      "cohen_kappa_score(INCIVILITY_DUMMY): 0.5346812616578949\n",
      "krippendorf(INCIVILITY_DUMMY): 0.534524430255902\n",
      "percent_agreement(INCIVILITY_DUMMY): 76.72%\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.83      0.59      0.69      1567\n",
      "           1       0.67      0.87      0.76      1522\n",
      "\n",
      "    accuracy                           0.73      3089\n",
      "   macro avg       0.75      0.73      0.73      3089\n",
      "weighted avg       0.75      0.73      0.73      3089\n",
      "\n",
      "cohen_kappa_score(INCIVILITY_DUMMY): 0.4628177124183961\n",
      "krippendorf(INCIVILITY_DUMMY): 0.45146406032973085\n",
      "percent_agreement(INCIVILITY_DUMMY): 73.03%\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.85      0.62      0.72      1567\n",
      "           1       0.69      0.89      0.78      1522\n",
      "\n",
      "    accuracy                           0.75      3089\n",
      "   macro avg       0.77      0.75      0.75      3089\n",
      "weighted avg       0.77      0.75      0.75      3089\n",
      "\n",
      "cohen_kappa_score(INCIVILITY_DUMMY): 0.5072169865098333\n",
      "krippendorf(INCIVILITY_DUMMY): 0.4977813629378637\n",
      "percent_agreement(INCIVILITY_DUMMY): 75.27%\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.87      0.71      0.78      2297\n",
      "           1       0.45      0.70      0.55       792\n",
      "\n",
      "    accuracy                           0.70      3089\n",
      "   macro avg       0.66      0.70      0.66      3089\n",
      "weighted avg       0.76      0.70      0.72      3089\n",
      "\n",
      "cohen_kappa_score(INTERACTIVITY_DUMMY): 0.3430756584425837\n",
      "krippendorf(INTERACTIVITY_DUMMY): 0.3283807435994265\n",
      "percent_agreement(INTERACTIVITY_DUMMY): 70.44%\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.89      0.61      0.72      2297\n",
      "           1       0.41      0.78      0.54       792\n",
      "\n",
      "    accuracy                           0.65      3089\n",
      "   macro avg       0.65      0.69      0.63      3089\n",
      "weighted avg       0.77      0.65      0.67      3089\n",
      "\n",
      "cohen_kappa_score(INTERACTIVITY_DUMMY): 0.2996431335699764\n",
      "krippendorf(INTERACTIVITY_DUMMY): 0.25834281956400373\n",
      "percent_agreement(INTERACTIVITY_DUMMY): 65.26%\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.86      0.62      0.72      2297\n",
      "           1       0.39      0.72      0.51       792\n",
      "\n",
      "    accuracy                           0.64      3089\n",
      "   macro avg       0.63      0.67      0.61      3089\n",
      "weighted avg       0.74      0.64      0.67      3089\n",
      "\n",
      "cohen_kappa_score(INTERACTIVITY_DUMMY): 0.26404842191866074\n",
      "krippendorf(INTERACTIVITY_DUMMY): 0.22857714543880758\n",
      "percent_agreement(INTERACTIVITY_DUMMY): 64.36%\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.93      0.85      0.89      2440\n",
      "           1       0.57      0.77      0.65       649\n",
      "\n",
      "    accuracy                           0.83      3089\n",
      "   macro avg       0.75      0.81      0.77      3089\n",
      "weighted avg       0.86      0.83      0.84      3089\n",
      "\n",
      "cohen_kappa_score(LIBERAL_DUMMY): 0.5435589327727111\n",
      "krippendorf(LIBERAL_DUMMY): 0.5404288426702988\n",
      "percent_agreement(LIBERAL_DUMMY): 82.94%\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.95      0.90      0.92      2611\n",
      "           1       0.57      0.73      0.64       478\n",
      "\n",
      "    accuracy                           0.87      3089\n",
      "   macro avg       0.76      0.81      0.78      3089\n",
      "weighted avg       0.89      0.87      0.88      3089\n",
      "\n",
      "cohen_kappa_score(CONSERVATIVE_DUMMY): 0.5615336292118079\n",
      "krippendorf(CONSERVATIVE_DUMMY): 0.5600409196001979\n",
      "percent_agreement(CONSERVATIVE_DUMMY): 87.15%\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.77      0.81      0.79      1505\n",
      "           1       0.81      0.76      0.79      1584\n",
      "\n",
      "    accuracy                           0.79      3089\n",
      "   macro avg       0.79      0.79      0.79      3089\n",
      "weighted avg       0.79      0.79      0.79      3089\n",
      "\n",
      "cohen_kappa_score(HAS_OPINION_DUMMY): 0.5749961205707755\n",
      "krippendorf(HAS_OPINION_DUMMY): 0.5746797355824025\n",
      "percent_agreement(HAS_OPINION_DUMMY): 78.73%\n"
     ]
    }
   ],
   "source": [
    "#evaluete respective performance on the TRAINING set (NOTE the wording of the few shot examples in the prompt is polished by GPT4o, not exactly like codebook):\n",
    "llm_human_column_pairs = [\n",
    "    (\"RATIONALITY_DUMMY\", \"rationality_combine_dum\"),\n",
    "    (\"RATIONALITY_DUMMY\", \"rationality_combine_exactexample_dum\"),\n",
    "    (\"RATIONALITY_DUMMY\", \"rationality_simple_dum\"),\n",
    "    (\"RATIONALITY_DUMMY\", \"rationality_simple2_dum\"),\n",
    "    (\"INCIVILITY_DUMMY\", \"incivility_combine_dum\"),\n",
    "    (\"INCIVILITY_DUMMY\", \"incivility_simple_dum\"),\n",
    "    (\"INCIVILITY_DUMMY\", \"incivility_simple2_dum\"),\n",
    "    (\"INTERACTIVITY_DUMMY\", \"interactivity_acknowledgement_simple_dum\"),\n",
    "    (\"INTERACTIVITY_DUMMY\", \"interactivity_acknowledgement_simple2_dum\"),\n",
    "    (\"LIBERAL_DUMMY\", \"political_liberal_US_dum\"),\n",
    "    (\"CONSERVATIVE_DUMMY\", \"political_conservative_US_dum\"),\n",
    "    (\"HAS_OPINION_DUMMY\", \"political_opinion_US_dum\")\n",
    "]\n",
    "print(\"Inter-coder reliablity: classification_report(human as gold, llm as predicted), cohen_kappa(human, llm):\")\n",
    "\n",
    "for human_col, llm_col in llm_human_column_pairs:\n",
    "    subset = dataset_w_pred_anon.loc[dataset_w_pred_anon.split == 'train', [human_col, llm_col]].dropna()\n",
    "    human = subset[human_col].tolist()\n",
    "    llm = subset[llm_col].tolist()\n",
    "\n",
    "    # Calculate percent agreement\n",
    "    agreement = np.sum(np.array(human) == np.array(llm))/len(np.array(human)) * 100\n",
    "\n",
    "    print(\"---\")\n",
    "    print(f\"{classification_report(human, llm)}\")\n",
    "    print(f\"cohen_kappa_score({human_col}): {cohen_kappa_score(human, llm)}\")\n",
    "    print(f\"krippendorf({human_col}): {krippendorff.alpha(np.array([human, llm]), level_of_measurement=\"nominal\")}\")\n",
    "    print(f\"percent_agreement({human_col}): {agreement:.2f}%\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "794f07a6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Inter-coder reliablity: classification_report(human as gold, llm as predicted), cohen_kappa(human, llm):\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.93      0.72      0.81      3165\n",
      "           1       0.38      0.77      0.51       697\n",
      "\n",
      "    accuracy                           0.73      3862\n",
      "   macro avg       0.66      0.74      0.66      3862\n",
      "weighted avg       0.83      0.73      0.76      3862\n",
      "\n",
      "cohen_kappa_score(RATIONALITY_DUMMY): 0.34728201946365866\n",
      "krippendorf(RATIONALITY_DUMMY): 0.31883884197869394\n",
      "percent_agreement(RATIONALITY_DUMMY): 72.92%\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.93      0.73      0.82      3165\n",
      "           1       0.38      0.76      0.51       697\n",
      "\n",
      "    accuracy                           0.74      3862\n",
      "   macro avg       0.66      0.75      0.67      3862\n",
      "weighted avg       0.83      0.74      0.76      3862\n",
      "\n",
      "cohen_kappa_score(RATIONALITY_DUMMY): 0.35617137578933\n",
      "krippendorf(RATIONALITY_DUMMY): 0.3302467199928376\n",
      "percent_agreement(RATIONALITY_DUMMY): 73.61%\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.88      0.88      0.88      3165\n",
      "           1       0.45      0.44      0.45       697\n",
      "\n",
      "    accuracy                           0.80      3862\n",
      "   macro avg       0.66      0.66      0.66      3862\n",
      "weighted avg       0.80      0.80      0.80      3862\n",
      "\n",
      "cohen_kappa_score(RATIONALITY_DUMMY): 0.32597649823629826\n",
      "krippendorf(RATIONALITY_DUMMY): 0.3260600046487714\n",
      "percent_agreement(RATIONALITY_DUMMY): 80.14%\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.87      0.96      0.91      3165\n",
      "           1       0.64      0.34      0.45       697\n",
      "\n",
      "    accuracy                           0.85      3862\n",
      "   macro avg       0.75      0.65      0.68      3862\n",
      "weighted avg       0.83      0.85      0.83      3862\n",
      "\n",
      "cohen_kappa_score(RATIONALITY_DUMMY): 0.3670917052423638\n",
      "krippendorf(RATIONALITY_DUMMY): 0.35784307942099336\n",
      "percent_agreement(RATIONALITY_DUMMY): 84.67%\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.78      0.75      0.77      1975\n",
      "           1       0.75      0.78      0.77      1887\n",
      "\n",
      "    accuracy                           0.77      3862\n",
      "   macro avg       0.77      0.77      0.77      3862\n",
      "weighted avg       0.77      0.77      0.77      3862\n",
      "\n",
      "cohen_kappa_score(INCIVILITY_DUMMY): 0.5346522710098467\n",
      "krippendorf(INCIVILITY_DUMMY): 0.5344976094230922\n",
      "percent_agreement(INCIVILITY_DUMMY): 76.72%\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.83      0.59      0.69      1975\n",
      "           1       0.67      0.87      0.76      1887\n",
      "\n",
      "    accuracy                           0.73      3862\n",
      "   macro avg       0.75      0.73      0.73      3862\n",
      "weighted avg       0.75      0.73      0.72      3862\n",
      "\n",
      "cohen_kappa_score(INCIVILITY_DUMMY): 0.4615900721451487\n",
      "krippendorf(INCIVILITY_DUMMY): 0.4501691366746845\n",
      "percent_agreement(INCIVILITY_DUMMY): 72.92%\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.85      0.63      0.72      1975\n",
      "           1       0.69      0.89      0.78      1887\n",
      "\n",
      "    accuracy                           0.75      3862\n",
      "   macro avg       0.77      0.76      0.75      3862\n",
      "weighted avg       0.77      0.75      0.75      3862\n",
      "\n",
      "cohen_kappa_score(INCIVILITY_DUMMY): 0.5092702345298536\n",
      "krippendorf(INCIVILITY_DUMMY): 0.5000776032423687\n",
      "percent_agreement(INCIVILITY_DUMMY): 75.32%\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.87      0.70      0.78      2856\n",
      "           1       0.45      0.70      0.55      1006\n",
      "\n",
      "    accuracy                           0.70      3862\n",
      "   macro avg       0.66      0.70      0.66      3862\n",
      "weighted avg       0.76      0.70      0.72      3862\n",
      "\n",
      "cohen_kappa_score(INTERACTIVITY_DUMMY): 0.3412070656517501\n",
      "krippendorf(INTERACTIVITY_DUMMY): 0.326841598641776\n",
      "percent_agreement(INTERACTIVITY_DUMMY): 70.22%\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.88      0.60      0.72      2856\n",
      "           1       0.41      0.77      0.53      1006\n",
      "\n",
      "    accuracy                           0.65      3862\n",
      "   macro avg       0.65      0.69      0.63      3862\n",
      "weighted avg       0.76      0.65      0.67      3862\n",
      "\n",
      "cohen_kappa_score(INTERACTIVITY_DUMMY): 0.2930463035681862\n",
      "krippendorf(INTERACTIVITY_DUMMY): 0.2521921135217433\n",
      "percent_agreement(INTERACTIVITY_DUMMY): 64.86%\n"
     ]
    }
   ],
   "source": [
    "#evaluate respective performance on full data (NOTE the wording of the few shot examples in the prompt is polished by GPT4o, not exactly like codebook):\n",
    "llm_human_column_pairs = [\n",
    "    (\"RATIONALITY_DUMMY\", \"rationality_combine_dum\"),\n",
    "    (\"RATIONALITY_DUMMY\", \"rationality_combine_exactexample_dum\"),\n",
    "    (\"RATIONALITY_DUMMY\", \"rationality_simple_dum\"),\n",
    "    (\"RATIONALITY_DUMMY\", \"rationality_simple2_dum\"),\n",
    "    (\"INCIVILITY_DUMMY\", \"incivility_combine_dum\"),\n",
    "    (\"INCIVILITY_DUMMY\", \"incivility_simple_dum\"),\n",
    "    (\"INCIVILITY_DUMMY\", \"incivility_simple2_dum\"),\n",
    "    (\"INTERACTIVITY_DUMMY\", \"interactivity_acknowledgement_simple_dum\"),\n",
    "    (\"INTERACTIVITY_DUMMY\", \"interactivity_acknowledgement_simple2_dum\")\n",
    "]\n",
    "print(\"Inter-coder reliablity: classification_report(human as gold, llm as predicted), cohen_kappa(human, llm):\")\n",
    "\n",
    "for human_col, llm_col in llm_human_column_pairs:\n",
    "    subset = dataset_w_pred_anon[[human_col, llm_col]].dropna()\n",
    "    human = subset[human_col].tolist()\n",
    "    llm = subset[llm_col].tolist()\n",
    "\n",
    "    # Calculate percent agreement\n",
    "    agreement = np.sum(np.array(human) == np.array(llm))/len(np.array(human)) * 100\n",
    "\n",
    "    print(\"---\")\n",
    "    print(f\"{classification_report(human, llm)}\")\n",
    "    print(f\"cohen_kappa_score({human_col}): {cohen_kappa_score(human, llm)}\")\n",
    "    print(f\"krippendorf({human_col}): {krippendorff.alpha(np.array([human, llm]), level_of_measurement=\"nominal\")}\")\n",
    "    print(f\"percent_agreement({human_col}): {agreement:.2f}%\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b9f3eecb",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "classifying interactivity_acknowledgement_simple: 100%|██████████| 3862/3862 [17:33<00:00,  3.67it/s]\n",
      "classifying rationality_simple2: 100%|██████████| 3862/3862 [16:50<00:00,  3.82it/s]\n"
     ]
    }
   ],
   "source": [
    "#what if we would use the lightweight version of the model:\n",
    "MODELsmall: str = 'llama3.1:8b-instruct-q6_K' # options: 'gemma:7b-instruct-q6_K', 'gemma2:27b-instruct-q6_K', 'llama3.1:8b-instruct-q6_K', 'llama3.1:70b-instruct-q6_K', 'mistral:7b-instruct-v0.3-q6_K', 'mistral-large:123b-instruct-2407-q6_K', 'mixtral:8x7b-instruct-v0.1-q6_K', 'mixtral:8x22b-instruct-v0.1-q6_K', 'phi3:14b-medium-128k-instruct-q6_K' or 'qwen2:72b-instruct-q6_K'\n",
    "\n",
    "pubspherepromptsrun25 = ['interactivity_acknowledgement_simple', 'political_ideology_US', 'incivility_simple2', 'rationality_simple2']   \n",
    "predictions21: typing.Dict[str, np.ndarray] = {\n",
    "    label: (\n",
    "        src.PromptClassify\n",
    "        .from_json(path)\n",
    "        (dataset[\"commentText\"], model=MODELsmall, options=options_low)\n",
    "    )\n",
    "    for label, path in CFG.prompt_classify_files.items() if label in pubspherepromptsrun25\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "56782018",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "classifying incivility_simple2: 100%|██████████| 3862/3862 [16:53<00:00,  3.81it/s]\n",
      "classifying political_ideology_US: 100%|██████████| 3862/3862 [18:16<00:00,  3.52it/s]\n"
     ]
    }
   ],
   "source": [
    "MODELsmall: str = 'llama3.1:8b-instruct-q6_K' # options: 'gemma:7b-instruct-q6_K', 'gemma2:27b-instruct-q6_K', 'llama3.1:8b-instruct-q6_K', 'llama3.1:70b-instruct-q6_K', 'mistral:7b-instruct-v0.3-q6_K', 'mistral-large:123b-instruct-2407-q6_K', 'mixtral:8x7b-instruct-v0.1-q6_K', 'mixtral:8x22b-instruct-v0.1-q6_K', 'phi3:14b-medium-128k-instruct-q6_K' or 'qwen2:72b-instruct-q6_K'\n",
    "\n",
    "pubspherepromptsrun25 = ['political_ideology_US', 'incivility_simple2']   \n",
    "predictions25: typing.Dict[str, np.ndarray] = {\n",
    "    label: (\n",
    "        src.PromptClassify\n",
    "        .from_json(path)\n",
    "        (dataset[\"commentText\"], model=MODELsmall, options=options_low)\n",
    "    )\n",
    "    for label, path in CFG.prompt_classify_files.items() if label in pubspherepromptsrun25\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "db4c04cb",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "classifying interactivity_acknowledgement_simple: 100%|██████████| 3862/3862 [16:41<00:00,  3.86it/s]\n"
     ]
    }
   ],
   "source": [
    "MODELsmall: str = 'llama3.1:8b-instruct-q6_K' # options: 'gemma:7b-instruct-q6_K', 'gemma2:27b-instruct-q6_K', 'llama3.1:8b-instruct-q6_K', 'llama3.1:70b-instruct-q6_K', 'mistral:7b-instruct-v0.3-q6_K', 'mistral-large:123b-instruct-2407-q6_K', 'mixtral:8x7b-instruct-v0.1-q6_K', 'mixtral:8x22b-instruct-v0.1-q6_K', 'phi3:14b-medium-128k-instruct-q6_K' or 'qwen2:72b-instruct-q6_K'\n",
    "\n",
    "pubspherepromptsrun24 = ['interactivity_acknowledgement_simple2']   \n",
    "predictions24: typing.Dict[str, np.ndarray] = {\n",
    "    label: (\n",
    "        src.PromptClassify\n",
    "        .from_json(path)\n",
    "        (dataset[\"commentText\"], model=MODELsmall, options=options_low)\n",
    "    )\n",
    "    for label, path in CFG.prompt_classify_files.items() if label in pubspherepromptsrun24\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "023a2c6b",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_w_pred_2 = pd.read_parquet(f'{CFG.report_dir}/publicsphere.cardiff_prompt_classify_s.parquet')\n",
    "dataset_w_pred_anon = pd.read_parquet('data/publicsphere/publicsphere.cardiff_prompt_classify_anon.parquet')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "47b4d0f0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "interactivity_acknowledgement_simple\n",
      "No     2040\n",
      "Yes    1697\n",
      "Name: count, dtype: int64\n",
      "------------------------------------------\n",
      "rationality_simple2\n",
      "No     3681\n",
      "Yes     155\n",
      "Name: count, dtype: int64\n",
      "------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "#join to the dataset:   \n",
    "for _, preds in predictions21.items():\n",
    "    print(preds.value_counts())\n",
    "    print(\"-\" * 42)\n",
    "     \n",
    "    dataset_w_pred_2 = dataset_w_pred_2.join(preds, rsuffix='_small')\n",
    "    dataset_w_pred_anon = dataset_w_pred_anon.join(preds, rsuffix='_small')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2bab1611",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "interactivity_acknowledgement_simple\n",
      "No     2391\n",
      "Yes    1422\n",
      "Name: count, dtype: int64\n",
      "------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "#join to the dataset:   \n",
    "for _, preds in predictions24.items():\n",
    "    print(preds.value_counts())\n",
    "    print(\"-\" * 42)\n",
    "     \n",
    "    dataset_w_pred_2 = dataset_w_pred_2.join(preds, rsuffix='_gpt4o2')\n",
    "    dataset_w_pred_anon = dataset_w_pred_anon.join(preds, rsuffix='_gpt4o2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "d1aa7e51",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "incivility_simple2\n",
      "No     2782\n",
      "Yes    1065\n",
      "Name: count, dtype: int64\n",
      "------------------------------------------\n",
      "political_ideology_US\n",
      "neutral         1901\n",
      "conservative     992\n",
      "liberal          647\n",
      "Name: count, dtype: int64\n",
      "------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "#join to the dataset:   \n",
    "for _, preds in predictions25.items():\n",
    "    print(preds.value_counts())\n",
    "    print(\"-\" * 42)\n",
    "     \n",
    "    dataset_w_pred_2 = dataset_w_pred_2.join(preds, rsuffix='_small')\n",
    "    dataset_w_pred_anon = dataset_w_pred_anon.join(preds, rsuffix='_small')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e683f6b9",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_w_pred_anon.loc[:, 'incivility_simple2_small_dum'] = dataset_w_pred_2.loc[:, 'incivility_simple2_small'].map({\"Yes\": 1, \"No\":0}).fillna(0).astype(int)\n",
    "dataset_w_pred_anon.loc[:, 'rationality_simple2_small_dum'] = dataset_w_pred_2.loc[:, 'rationality_simple2_small'].map({\"Yes\": 1, \"No\":0}).fillna(0).astype(int)\n",
    "dataset_w_pred_anon.loc[:, 'interactivity_acknowledgement_simple_small_dum'] = dataset_w_pred_2.loc[:, 'interactivity_acknowledgement_simple_small'].map({\"Yes\": 1, \"No\":0}).fillna(0).astype(int)\n",
    "dataset_w_pred_anon.loc[:, 'interactivity_acknowledgement_simple_small2_dum'] = dataset_w_pred_2.loc[:, 'interactivity_acknowledgement_simple_small2'].map({\"Yes\": 1, \"No\":0}).fillna(0).astype(int)\n",
    "\n",
    "dataset_w_pred_anon.loc[:, 'political_conservative_US_small_dum'] = dataset_w_pred_anon.loc[:, 'political_ideology_US'].map({\"conservative\": 1}).fillna(0).astype(int)\n",
    "dataset_w_pred_anon.loc[:, 'political_liberal_US_small_dum'] = dataset_w_pred_anon.loc[:, 'political_ideology_US'].map({\"liberal\": 1}).fillna(0).astype(int)\n",
    "dataset_w_pred_anon.loc[:, 'political_opinion_US_small_dum'] = dataset_w_pred_anon.loc[:, 'political_ideology_US'].map({\"liberal\": 1, 'conservative': 1}).fillna(0).astype(int)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "d750765c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Inter-coder reliablity: classification_report(human as gold, llm as predicted), cohen_kappa(human, llm):\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.84      0.98      0.91      2541\n",
      "           1       0.66      0.15      0.24       548\n",
      "\n",
      "    accuracy                           0.83      3089\n",
      "   macro avg       0.75      0.56      0.57      3089\n",
      "weighted avg       0.81      0.83      0.79      3089\n",
      "\n",
      "cohen_kappa_score(RATIONALITY_DUMMY): 0.18623102838033445\n",
      "krippendorf(RATIONALITY_DUMMY): 0.14635157545605315\n",
      "percent_agreement(RATIONALITY_DUMMY): 83.49%\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.65      0.93      0.77      1567\n",
      "           1       0.87      0.50      0.63      1522\n",
      "\n",
      "    accuracy                           0.72      3089\n",
      "   macro avg       0.76      0.71      0.70      3089\n",
      "weighted avg       0.76      0.72      0.70      3089\n",
      "\n",
      "cohen_kappa_score(INCIVILITY_DUMMY): 0.42785894170415706\n",
      "krippendorf(INCIVILITY_DUMMY): 0.40057707407132315\n",
      "percent_agreement(INCIVILITY_DUMMY): 71.58%\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.83      0.63      0.71      2297\n",
      "           1       0.36      0.62      0.46       792\n",
      "\n",
      "    accuracy                           0.63      3089\n",
      "   macro avg       0.60      0.62      0.59      3089\n",
      "weighted avg       0.71      0.63      0.65      3089\n",
      "\n",
      "cohen_kappa_score(INTERACTIVITY_DUMMY): 0.20088682469320962\n",
      "krippendorf(INTERACTIVITY_DUMMY): 0.17310530790213474\n",
      "percent_agreement(INTERACTIVITY_DUMMY): 62.61%\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.81      0.69      0.75      2297\n",
      "           1       0.37      0.54      0.44       792\n",
      "\n",
      "    accuracy                           0.65      3089\n",
      "   macro avg       0.59      0.61      0.59      3089\n",
      "weighted avg       0.70      0.65      0.67      3089\n",
      "\n",
      "cohen_kappa_score(INTERACTIVITY_DUMMY): 0.1968562703223654\n",
      "krippendorf(INTERACTIVITY_DUMMY): 0.1853250989071804\n",
      "percent_agreement(INTERACTIVITY_DUMMY): 65.00%\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.88      0.92      0.90      2440\n",
      "           1       0.64      0.51      0.57       649\n",
      "\n",
      "    accuracy                           0.84      3089\n",
      "   macro avg       0.76      0.72      0.73      3089\n",
      "weighted avg       0.83      0.84      0.83      3089\n",
      "\n",
      "cohen_kappa_score(LIBERAL_DUMMY): 0.46797069443461115\n",
      "krippendorf(LIBERAL_DUMMY): 0.46656938641762924\n",
      "percent_agreement(LIBERAL_DUMMY): 83.62%\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.92      0.81      0.86      2611\n",
      "           1       0.37      0.60      0.46       478\n",
      "\n",
      "    accuracy                           0.78      3089\n",
      "   macro avg       0.64      0.71      0.66      3089\n",
      "weighted avg       0.83      0.78      0.80      3089\n",
      "\n",
      "cohen_kappa_score(CONSERVATIVE_DUMMY): 0.32794156824438336\n",
      "krippendorf(CONSERVATIVE_DUMMY): 0.31778778535107655\n",
      "percent_agreement(CONSERVATIVE_DUMMY): 77.79%\n"
     ]
    }
   ],
   "source": [
    "#evaluate performance of small model on the TRAINING set:\n",
    "llm_human_column_pairs = [\n",
    "    (\"RATIONALITY_DUMMY\", \"rationality_simple2_small_dum\"),\n",
    "    (\"INCIVILITY_DUMMY\", \"incivility_simple2_small_dum\"),\n",
    "    (\"INTERACTIVITY_DUMMY\", \"interactivity_acknowledgement_simple_small_dum\"),\n",
    "    (\"INTERACTIVITY_DUMMY\", \"interactivity_acknowledgement_simple_small2_dum\"),\n",
    "    (\"LIBERAL_DUMMY\", \"political_liberal_US_small_dum\"),\n",
    "    (\"CONSERVATIVE_DUMMY\", \"political_conservative_US_small_dum\")\n",
    "]\n",
    "print(\"Inter-coder reliablity: classification_report(human as gold, llm as predicted), cohen_kappa(human, llm):\")\n",
    "\n",
    "for human_col, llm_col in llm_human_column_pairs:\n",
    "    subset = dataset_w_pred_anon.loc[dataset_w_pred_anon.split == 'train', [human_col, llm_col]].dropna()\n",
    "    human = subset[human_col].tolist()\n",
    "    llm = subset[llm_col].tolist()\n",
    "\n",
    "    # Calculate percent agreement\n",
    "    agreement = np.sum(np.array(human) == np.array(llm))/len(np.array(human)) * 100\n",
    "\n",
    "    print(\"---\")\n",
    "    print(f\"{classification_report(human, llm)}\")\n",
    "    print(f\"cohen_kappa_score({human_col}): {cohen_kappa_score(human, llm)}\")\n",
    "    print(f\"krippendorf({human_col}): {krippendorff.alpha(np.array([human, llm]), level_of_measurement=\"nominal\")}\")\n",
    "    print(f\"percent_agreement({human_col}): {agreement:.2f}%\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "11fabbad",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Inter-coder reliablity: classification_report(human as gold, llm as predicted), cohen_kappa(human, llm):\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.85      0.94      0.89       624\n",
      "           1       0.56      0.30      0.39       149\n",
      "\n",
      "    accuracy                           0.82       773\n",
      "   macro avg       0.70      0.62      0.64       773\n",
      "weighted avg       0.79      0.82      0.80       773\n",
      "\n",
      "cohen_kappa_score(RATIONALITY_DUMMY): 0.2912993935901299\n",
      "krippendorf(RATIONALITY_DUMMY): 0.2802092484625829\n",
      "percent_agreement(RATIONALITY_DUMMY): 81.89%\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.86      0.64      0.74       408\n",
      "           1       0.69      0.88      0.77       365\n",
      "\n",
      "    accuracy                           0.76       773\n",
      "   macro avg       0.77      0.76      0.75       773\n",
      "weighted avg       0.78      0.76      0.75       773\n",
      "\n",
      "cohen_kappa_score(INCIVILITY_DUMMY): 0.5165280641200862\n",
      "krippendorf(INCIVILITY_DUMMY): 0.5085456060185303\n",
      "percent_agreement(INCIVILITY_DUMMY): 75.55%\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.85      0.70      0.77       559\n",
      "           1       0.46      0.69      0.55       214\n",
      "\n",
      "    accuracy                           0.69       773\n",
      "   macro avg       0.66      0.69      0.66       773\n",
      "weighted avg       0.75      0.69      0.71       773\n",
      "\n",
      "cohen_kappa_score(INTERACTIVITY_DUMMY): 0.33329815457063106\n",
      "krippendorf(INTERACTIVITY_DUMMY): 0.32061451114636386\n",
      "percent_agreement(INTERACTIVITY_DUMMY): 69.34%\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.95      0.85      0.90       633\n",
      "           1       0.54      0.81      0.65       140\n",
      "\n",
      "    accuracy                           0.84       773\n",
      "   macro avg       0.75      0.83      0.77       773\n",
      "weighted avg       0.88      0.84      0.85       773\n",
      "\n",
      "cohen_kappa_score(LIBERAL_DUMMY): 0.5473341204250295\n",
      "krippendorf(LIBERAL_DUMMY): 0.5423315814620162\n",
      "percent_agreement(LIBERAL_DUMMY): 83.96%\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.96      0.91      0.93       660\n",
      "           1       0.60      0.80      0.68       113\n",
      "\n",
      "    accuracy                           0.89       773\n",
      "   macro avg       0.78      0.85      0.81       773\n",
      "weighted avg       0.91      0.89      0.90       773\n",
      "\n",
      "cohen_kappa_score(CONSERVATIVE_DUMMY): 0.6179256940439904\n",
      "krippendorf(CONSERVATIVE_DUMMY): 0.6165437526591973\n",
      "percent_agreement(CONSERVATIVE_DUMMY): 89.13%\n"
     ]
    }
   ],
   "source": [
    "#evaluate performance of BEST prompts on the TEST set:\n",
    "llm_human_column_pairs = [\n",
    "    (\"RATIONALITY_DUMMY\", \"rationality_simple2_dum\"),\n",
    "    (\"INCIVILITY_DUMMY\", \"incivility_simple2_dum\"),\n",
    "    (\"INTERACTIVITY_DUMMY\", \"interactivity_acknowledgement_simple_dum\"),\n",
    "    (\"LIBERAL_DUMMY\", \"political_liberal_US_dum\"),\n",
    "    (\"CONSERVATIVE_DUMMY\", \"political_conservative_US_dum\")\n",
    "]\n",
    "\n",
    "print(\"Inter-coder reliablity: classification_report(human as gold, llm as predicted), cohen_kappa(human, llm):\")\n",
    "\n",
    "for human_col, llm_col in llm_human_column_pairs:\n",
    "    subset = dataset_w_pred_anon.loc[dataset_w_pred_anon.split == 'test', [human_col, llm_col]].dropna()\n",
    "    human = subset[human_col].tolist()\n",
    "    llm = subset[llm_col].tolist()\n",
    "\n",
    "    # Calculate percent agreement\n",
    "    agreement = np.sum(np.array(human) == np.array(llm))/len(np.array(human)) * 100\n",
    "\n",
    "    print(\"---\")\n",
    "    print(f\"{classification_report(human, llm)}\")\n",
    "    print(f\"cohen_kappa_score({human_col}): {cohen_kappa_score(human, llm)}\")\n",
    "    print(f\"krippendorf({human_col}): {krippendorff.alpha(np.array([human, llm]), level_of_measurement=\"nominal\")}\")\n",
    "    print(f\"percent_agreement({human_col}): {agreement:.2f}%\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f31ece0a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Inter-coder reliablity: classification_report(human as gold, llm as predicted), cohen_kappa(human, llm):\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.84      0.98      0.91      3165\n",
      "           1       0.65      0.14      0.24       697\n",
      "\n",
      "    accuracy                           0.83      3862\n",
      "   macro avg       0.75      0.56      0.57      3862\n",
      "weighted avg       0.81      0.83      0.78      3862\n",
      "\n",
      "cohen_kappa_score(RATIONALITY_DUMMY): 0.18347073889343912\n",
      "krippendorf(RATIONALITY_DUMMY): 0.14261349041083915\n",
      "percent_agreement(RATIONALITY_DUMMY): 83.17%\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.66      0.93      0.77      1975\n",
      "           1       0.88      0.49      0.63      1887\n",
      "\n",
      "    accuracy                           0.72      3862\n",
      "   macro avg       0.77      0.71      0.70      3862\n",
      "weighted avg       0.76      0.72      0.70      3862\n",
      "\n",
      "cohen_kappa_score(INCIVILITY_DUMMY): 0.43074417373383656\n",
      "krippendorf(INCIVILITY_DUMMY): 0.4035169018915671\n",
      "percent_agreement(INCIVILITY_DUMMY): 71.83%\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.82      0.62      0.71      2856\n",
      "           1       0.37      0.62      0.46      1006\n",
      "\n",
      "    accuracy                           0.62      3862\n",
      "   macro avg       0.60      0.62      0.59      3862\n",
      "weighted avg       0.71      0.62      0.65      3862\n",
      "\n",
      "cohen_kappa_score(INTERACTIVITY_DUMMY): 0.202268534876325\n",
      "krippendorf(INTERACTIVITY_DUMMY): 0.17430970464190987\n",
      "percent_agreement(INTERACTIVITY_DUMMY): 62.43%\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.81      0.69      0.74      2856\n",
      "           1       0.37      0.53      0.44      1006\n",
      "\n",
      "    accuracy                           0.65      3862\n",
      "   macro avg       0.59      0.61      0.59      3862\n",
      "weighted avg       0.69      0.65      0.66      3862\n",
      "\n",
      "cohen_kappa_score(INTERACTIVITY_DUMMY): 0.19036285550115162\n",
      "krippendorf(INTERACTIVITY_DUMMY): 0.179572752678967\n",
      "percent_agreement(INTERACTIVITY_DUMMY): 64.63%\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.88      0.92      0.90      3073\n",
      "           1       0.63      0.52      0.57       789\n",
      "\n",
      "    accuracy                           0.84      3862\n",
      "   macro avg       0.76      0.72      0.73      3862\n",
      "weighted avg       0.83      0.84      0.83      3862\n",
      "\n",
      "cohen_kappa_score(LIBERAL_DUMMY): 0.4708263183926331\n",
      "krippendorf(LIBERAL_DUMMY): 0.4697132797493745\n",
      "percent_agreement(LIBERAL_DUMMY): 83.95%\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.92      0.81      0.86      3271\n",
      "           1       0.37      0.62      0.46       591\n",
      "\n",
      "    accuracy                           0.78      3862\n",
      "   macro avg       0.64      0.71      0.66      3862\n",
      "weighted avg       0.84      0.78      0.80      3862\n",
      "\n",
      "cohen_kappa_score(CONSERVATIVE_DUMMY): 0.33171247451427865\n",
      "krippendorf(CONSERVATIVE_DUMMY): 0.32074610518883306\n",
      "percent_agreement(CONSERVATIVE_DUMMY): 77.86%\n"
     ]
    }
   ],
   "source": [
    "#evaluate performance small model on the full data:\n",
    "llm_human_column_pairs = [\n",
    "    (\"RATIONALITY_DUMMY\", \"rationality_simple2_small_dum\"),\n",
    "    (\"INCIVILITY_DUMMY\", \"incivility_simple2_small_dum\"),\n",
    "    (\"INTERACTIVITY_DUMMY\", \"interactivity_acknowledgement_simple_small_dum\"),\n",
    "    (\"INTERACTIVITY_DUMMY\", \"interactivity_acknowledgement_simple_small2_dum\"),\n",
    "    (\"LIBERAL_DUMMY\", \"political_liberal_US_small_dum\"),\n",
    "    (\"CONSERVATIVE_DUMMY\", \"political_conservative_US_small_dum\")\n",
    "]\n",
    "print(\"Inter-coder reliablity: classification_report(human as gold, llm as predicted), cohen_kappa(human, llm):\")\n",
    "\n",
    "for human_col, llm_col in llm_human_column_pairs:\n",
    "    subset = dataset_w_pred_anon[[human_col, llm_col]].dropna()\n",
    "    human = subset[human_col].tolist()\n",
    "    llm = subset[llm_col].tolist()\n",
    "\n",
    "    # Calculate percent agreement\n",
    "    agreement = np.sum(np.array(human) == np.array(llm))/len(np.array(human)) * 100\n",
    "\n",
    "    print(\"---\")\n",
    "    print(f\"{classification_report(human, llm)}\")\n",
    "    print(f\"cohen_kappa_score({human_col}): {cohen_kappa_score(human, llm)}\")\n",
    "    print(f\"krippendorf({human_col}): {krippendorff.alpha(np.array([human, llm]), level_of_measurement=\"nominal\")}\")\n",
    "    print(f\"percent_agreement({human_col}): {agreement:.2f}%\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "19ab2330",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Inter-coder reliablity: classification_report(human as gold, llm as predicted), cohen_kappa(human, llm):\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.94      0.85      0.89      3073\n",
      "           1       0.56      0.77      0.65       789\n",
      "\n",
      "    accuracy                           0.83      3862\n",
      "   macro avg       0.75      0.81      0.77      3862\n",
      "weighted avg       0.86      0.83      0.84      3862\n",
      "\n",
      "cohen_kappa_score(LIBERAL_DUMMY): 0.5444043080556233\n",
      "krippendorf(LIBERAL_DUMMY): 0.5408921571554941\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.95      0.90      0.92      3271\n",
      "           1       0.57      0.74      0.65       591\n",
      "\n",
      "    accuracy                           0.88      3862\n",
      "   macro avg       0.76      0.82      0.79      3862\n",
      "weighted avg       0.89      0.88      0.88      3862\n",
      "\n",
      "cohen_kappa_score(CONSERVATIVE_DUMMY): 0.5725715490094205\n",
      "krippendorf(CONSERVATIVE_DUMMY): 0.571045663399131\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.77      0.82      0.79      1907\n",
      "           1       0.81      0.77      0.79      1955\n",
      "\n",
      "    accuracy                           0.79      3862\n",
      "   macro avg       0.79      0.79      0.79      3862\n",
      "weighted avg       0.79      0.79      0.79      3862\n",
      "\n",
      "cohen_kappa_score(HAS_OPINION_DUMMY): 0.5843646306803332\n",
      "krippendorf(HAS_OPINION_DUMMY): 0.5841227804229595\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      0.47      0.64      3165\n",
      "           1       0.29      0.96      0.44       697\n",
      "\n",
      "    accuracy                           0.56      3862\n",
      "   macro avg       0.63      0.72      0.54      3862\n",
      "weighted avg       0.86      0.56      0.60      3862\n",
      "\n",
      "cohen_kappa_score(RATIONALITY_DUMMY): 0.22526101743876636\n",
      "krippendorf(RATIONALITY_DUMMY): 0.07797120095285182\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.83      0.69      0.76      1551\n",
      "           1       0.81      0.91      0.86      2311\n",
      "\n",
      "    accuracy                           0.82      3862\n",
      "   macro avg       0.82      0.80      0.81      3862\n",
      "weighted avg       0.82      0.82      0.82      3862\n",
      "\n",
      "cohen_kappa_score(TopicRelevance): 0.6157849447917124\n",
      "krippendorf(TopicRelevance): 0.6138587759369105\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.84      0.73      0.78      2856\n",
      "           1       0.44      0.61      0.51      1006\n",
      "\n",
      "    accuracy                           0.70      3862\n",
      "   macro avg       0.64      0.67      0.64      3862\n",
      "weighted avg       0.74      0.70      0.71      3862\n",
      "\n",
      "cohen_kappa_score(INTERACTIVITY_DUMMY): 0.29659271996536163\n",
      "krippendorf(INTERACTIVITY_DUMMY): 0.28843817944487826\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.90      0.47      0.62      1975\n",
      "           1       0.63      0.94      0.76      1887\n",
      "\n",
      "    accuracy                           0.70      3862\n",
      "   macro avg       0.76      0.71      0.69      3862\n",
      "weighted avg       0.77      0.70      0.69      3862\n",
      "\n",
      "cohen_kappa_score(INCIVILITY_DUMMY): 0.4112037776897317\n",
      "krippendorf(INCIVILITY_DUMMY): 0.3745362005182755\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.96      0.73      0.83      3004\n",
      "           1       0.49      0.90      0.63       858\n",
      "\n",
      "    accuracy                           0.77      3862\n",
      "   macro avg       0.72      0.81      0.73      3862\n",
      "weighted avg       0.86      0.77      0.79      3862\n",
      "\n",
      "cohen_kappa_score(HATELIST_FOCUSED_DUMMY): 0.4823754802996545\n",
      "krippendorf(HATELIST_FOCUSED_DUMMY): 0.46161414511028476\n"
     ]
    }
   ],
   "source": [
    "#These are the legacy results of the old prompts:\n",
    "\n",
    "print(\"Inter-coder reliablity: classification_report(human as gold, llm as predicted), cohen_kappa(human, llm):\")\n",
    "\n",
    "for human_col, llm_col in llm_human_column_pairs:\n",
    "    subset = dataset_w_pred_anon[[human_col, llm_col]].dropna()\n",
    "    human = subset[human_col].tolist()\n",
    "    llm = subset[llm_col].tolist()\n",
    "    \n",
    "    print(\"---\")\n",
    "    print(f\"{classification_report(human, llm)}\")\n",
    "    print(f\"cohen_kappa_score({human_col}): {cohen_kappa_score(human, llm)}\")\n",
    "    print(f\"krippendorf({human_col}): {krippendorff.alpha(np.array([human, llm]), level_of_measurement=\"nominal\")}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "657059b7",
   "metadata": {},
   "outputs": [],
   "source": [
    "MODELS: typing.Tuple[str, str] = [\n",
    "    (\"topics\", \"cardiffnlp/tweet-topic-21-multi\"),\n",
    "    (\"emotions\", \"cardiffnlp/twitter-roberta-base-emotion-multilabel-latest\"),\n",
    "    (\"sentiment\", \"cardiffnlp/twitter-roberta-base-sentiment-latest\"),\n",
    "    (\"irony\", \"cardiffnlp/twitter-roberta-base-irony\"),\n",
    "    (\"offensive\",  \"cardiffnlp/twitter-roberta-base-offensive\"),\n",
    "    (\"hate\", \"cardiffnlp/twitter-roberta-base-hate-latest\")\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "a445c413",
   "metadata": {},
   "outputs": [],
   "source": [
    "Cardiff_predictions: typing.Dict[str, pd.Series] = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "c4761f29",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 121/121 [07:09<00:00,  3.55s/it]\n",
      "100%|██████████| 121/121 [07:06<00:00,  3.52s/it]\n",
      "Some weights of the model checkpoint at cardiffnlp/twitter-roberta-base-sentiment-latest were not used when initializing RobertaForSequenceClassification: ['roberta.pooler.dense.bias', 'roberta.pooler.dense.weight']\n",
      "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "100%|██████████| 121/121 [07:03<00:00,  3.50s/it]\n",
      "100%|██████████| 121/121 [07:03<00:00,  3.50s/it]\n",
      "100%|██████████| 121/121 [07:07<00:00,  3.53s/it]\n",
      "100%|██████████| 121/121 [07:04<00:00,  3.50s/it]\n"
     ]
    }
   ],
   "source": [
    "for label, model in MODELS:\n",
    "    classifier = src.HFClassify(slug=model)\n",
    "    Cardiff_predictions[label] = pd.Series(classifier(dataset[\"commentText\"], theta=0.4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "416eb83a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'topics': 0                                                      {}\n",
       " 1                                 {news_&_social_concern}\n",
       " 2                                 {news_&_social_concern}\n",
       " 3                                 {news_&_social_concern}\n",
       " 4                                  {diaries_&_daily_life}\n",
       "                               ...                        \n",
       " 3857    {news_&_social_concern, business_&_entrepreneu...\n",
       " 3858                              {news_&_social_concern}\n",
       " 3859             {film_tv_&_video, news_&_social_concern}\n",
       " 3860                              {news_&_social_concern}\n",
       " 3861                              {news_&_social_concern}\n",
       " Length: 3862, dtype: object,\n",
       " 'emotions': 0                  {pessimism, sadness}\n",
       " 1                      {disgust, anger}\n",
       " 2        {anticipation, anger, disgust}\n",
       " 3                             {disgust}\n",
       " 4                             {sadness}\n",
       "                      ...               \n",
       " 3857                   {disgust, anger}\n",
       " 3858                     {anticipation}\n",
       " 3859                     {anticipation}\n",
       " 3860    {anticipation, optimism, trust}\n",
       " 3861             {disgust, anger, fear}\n",
       " Length: 3862, dtype: object,\n",
       " 'sentiment': 0       {neutral, negative}\n",
       " 1       {neutral, negative}\n",
       " 2       {neutral, negative}\n",
       " 3       {neutral, negative}\n",
       " 4       {neutral, positive}\n",
       "                ...         \n",
       " 3857    {neutral, negative}\n",
       " 3858              {neutral}\n",
       " 3859    {neutral, negative}\n",
       " 3860    {neutral, positive}\n",
       " 3861    {neutral, negative}\n",
       " Length: 3862, dtype: object,\n",
       " 'irony': 0       {non_irony, irony}\n",
       " 1              {non_irony}\n",
       " 2              {non_irony}\n",
       " 3                  {irony}\n",
       " 4       {non_irony, irony}\n",
       "                ...        \n",
       " 3857           {non_irony}\n",
       " 3858           {non_irony}\n",
       " 3859               {irony}\n",
       " 3860           {non_irony}\n",
       " 3861               {irony}\n",
       " Length: 3862, dtype: object,\n",
       " 'offensive': 0       {offensive, non-offensive}\n",
       " 1       {offensive, non-offensive}\n",
       " 2                  {non-offensive}\n",
       " 3                  {non-offensive}\n",
       " 4                  {non-offensive}\n",
       "                    ...            \n",
       " 3857               {non-offensive}\n",
       " 3858               {non-offensive}\n",
       " 3859               {non-offensive}\n",
       " 3860               {non-offensive}\n",
       " 3861               {non-offensive}\n",
       " Length: 3862, dtype: object,\n",
       " 'hate': 0       {NOT-HATE}\n",
       " 1       {NOT-HATE}\n",
       " 2       {NOT-HATE}\n",
       " 3       {NOT-HATE}\n",
       " 4       {NOT-HATE}\n",
       "            ...    \n",
       " 3857        {HATE}\n",
       " 3858    {NOT-HATE}\n",
       " 3859    {NOT-HATE}\n",
       " 3860    {NOT-HATE}\n",
       " 3861    {NOT-HATE}\n",
       " Length: 3862, dtype: object}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Cardiff_predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "68efb3ee",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'topics'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "{news_&_social_concern}                                                                        2161\n",
       "{diaries_&_daily_life}                                                                          240\n",
       "{}                                                                                              208\n",
       "{film_tv_&_video, news_&_social_concern}                                                        146\n",
       "{diaries_&_daily_life, news_&_social_concern}                                                    97\n",
       "                                                                                               ... \n",
       "{diaries_&_daily_life, news_&_social_concern, music}                                              1\n",
       "{film_tv_&_video, arts_&_culture}                                                                 1\n",
       "{news_&_social_concern, youth_&_student_life, sports, learning_&_educational}                     1\n",
       "{film_tv_&_video, news_&_social_concern, business_&_entrepreneurs, celebrity_&_pop_culture}       1\n",
       "{news_&_social_concern, business_&_entrepreneurs, fashion_&_style}                                1\n",
       "Name: count, Length: 116, dtype: int64"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'emotions'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "{disgust, anger}                       1481\n",
       "{disgust, anger, sadness}               278\n",
       "{anticipation}                          263\n",
       "{}                                      210\n",
       "{joy, optimism}                         200\n",
       "                                       ... \n",
       "{optimism, fear, sadness}                 1\n",
       "{anticipation, fear, disgust}             1\n",
       "{disgust, pessimism, fear, sadness}       1\n",
       "{fear, surprise}                          1\n",
       "{anticipation, joy, anger}                1\n",
       "Name: count, Length: 86, dtype: int64"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'sentiment'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "{neutral, negative}              2508\n",
       "{neutral, positive}               630\n",
       "{neutral}                         266\n",
       "{negative}                        239\n",
       "{positive}                        185\n",
       "{neutral, negative, positive}      29\n",
       "{negative, positive}                5\n",
       "Name: count, dtype: int64"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'irony'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "{non_irony, irony}    1874\n",
       "{non_irony}           1042\n",
       "{irony}                946\n",
       "Name: count, dtype: int64"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'offensive'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "{non-offensive}               2066\n",
       "{offensive, non-offensive}    1158\n",
       "{offensive}                    638\n",
       "Name: count, dtype: int64"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'hate'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "{NOT-HATE}          3532\n",
       "{HATE}               276\n",
       "{HATE, NOT-HATE}      54\n",
       "Name: count, dtype: int64"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "for label, series in Cardiff_predictions.items():\n",
    "    display(label)\n",
    "    display(series.value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d368b736",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_w_pred = pd.read_parquet(f'{CFG.report_dir}/publicsphere.cardiff_prompt_classify.parquet')\n",
    "\n",
    "for labels, preds in Cardiff_predictions.items():\n",
    "    dataset_w_pred_2 = dataset_w_pred.join(preds.rename(labels))\n",
    "\n",
    "dataset_w_pred_2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "7035666d-1db0-4d5d-aafe-2068bab80aef",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>StartDate</th>\n",
       "      <th>RecordedDate</th>\n",
       "      <th>IPAddress</th>\n",
       "      <th>Finished</th>\n",
       "      <th>Coder</th>\n",
       "      <th>ID</th>\n",
       "      <th>Mark_ID</th>\n",
       "      <th>Genre</th>\n",
       "      <th>topiccode</th>\n",
       "      <th>Platform</th>\n",
       "      <th>...</th>\n",
       "      <th>rationality_topic_relevance</th>\n",
       "      <th>political_negativity</th>\n",
       "      <th>rationality_background_info</th>\n",
       "      <th>rationality_reasoning</th>\n",
       "      <th>topics</th>\n",
       "      <th>emotions</th>\n",
       "      <th>sentiment</th>\n",
       "      <th>irony</th>\n",
       "      <th>offensive</th>\n",
       "      <th>hate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5/30/2021 13:03:17</td>\n",
       "      <td>5/30/2021 13:04:17</td>\n",
       "      <td>62.194.51.29</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>UgyPHwv8G0cDE6-wEgl4AaABAg.8_0ZjJKSJty8_0kXGkAd2U</td>\n",
       "      <td>119</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>No</td>\n",
       "      <td>not political/negative</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>{}</td>\n",
       "      <td>{pessimism, sadness}</td>\n",
       "      <td>{neutral, negative}</td>\n",
       "      <td>{non_irony, irony}</td>\n",
       "      <td>{offensive, non-offensive}</td>\n",
       "      <td>{NOT-HATE}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>10/11/2021 10:34:05</td>\n",
       "      <td>10/11/2021 10:36:46</td>\n",
       "      <td>213.127.109.191</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>Ugx2WXq9UdV8mPPjejJ4AaABAg.8yHCKV0Boe58yYRxEQEF45</td>\n",
       "      <td>282</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>Yes</td>\n",
       "      <td>not political/negative</td>\n",
       "      <td>No</td>\n",
       "      <td>Yes</td>\n",
       "      <td>{news_&amp;_social_concern}</td>\n",
       "      <td>{disgust, anger}</td>\n",
       "      <td>{neutral, negative}</td>\n",
       "      <td>{non_irony}</td>\n",
       "      <td>{offensive, non-offensive}</td>\n",
       "      <td>{NOT-HATE}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>9/9/2021 18:49:48</td>\n",
       "      <td>9/9/2021 18:51:32</td>\n",
       "      <td>213.127.110.0</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>1110578710648890000</td>\n",
       "      <td>372</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>Yes</td>\n",
       "      <td>political/negative</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>{news_&amp;_social_concern}</td>\n",
       "      <td>{anticipation, anger, disgust}</td>\n",
       "      <td>{neutral, negative}</td>\n",
       "      <td>{non_irony}</td>\n",
       "      <td>{non-offensive}</td>\n",
       "      <td>{NOT-HATE}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6/6/2021 16:12:46</td>\n",
       "      <td>6/6/2021 16:16:16</td>\n",
       "      <td>213.127.76.145</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>UgwUPFScjJ0MCeaP2F54AaABAg.8lvp3fc9Euf8lvvgsUgEgV</td>\n",
       "      <td>769</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>Yes</td>\n",
       "      <td>political/negative</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>{news_&amp;_social_concern}</td>\n",
       "      <td>{disgust}</td>\n",
       "      <td>{neutral, negative}</td>\n",
       "      <td>{irony}</td>\n",
       "      <td>{non-offensive}</td>\n",
       "      <td>{NOT-HATE}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6/13/2021 13:25:49</td>\n",
       "      <td>6/13/2021 13:27:28</td>\n",
       "      <td>213.127.82.232</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>UgwWKCWtSJdFvjGHvTp4AaABAg.8kUC5dGrQ2H8kUDRihE2f3</td>\n",
       "      <td>1206</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>No</td>\n",
       "      <td>not political/negative</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>{diaries_&amp;_daily_life}</td>\n",
       "      <td>{sadness}</td>\n",
       "      <td>{neutral, positive}</td>\n",
       "      <td>{non_irony, irony}</td>\n",
       "      <td>{non-offensive}</td>\n",
       "      <td>{NOT-HATE}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3857</th>\n",
       "      <td>8/19/2021 14:50:13</td>\n",
       "      <td>8/19/2021 14:54:28</td>\n",
       "      <td>62.194.51.29</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>1152219467579100000</td>\n",
       "      <td>10000695</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>Yes</td>\n",
       "      <td>political/negative</td>\n",
       "      <td>No</td>\n",
       "      <td>Yes</td>\n",
       "      <td>{news_&amp;_social_concern, business_&amp;_entrepreneu...</td>\n",
       "      <td>{disgust, anger}</td>\n",
       "      <td>{neutral, negative}</td>\n",
       "      <td>{non_irony}</td>\n",
       "      <td>{non-offensive}</td>\n",
       "      <td>{HATE}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3858</th>\n",
       "      <td>8/19/2021 15:10:27</td>\n",
       "      <td>8/19/2021 15:12:21</td>\n",
       "      <td>62.194.51.29</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>1085362296472430000</td>\n",
       "      <td>10007008</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>Yes</td>\n",
       "      <td>political/negative</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>{news_&amp;_social_concern}</td>\n",
       "      <td>{anticipation}</td>\n",
       "      <td>{neutral}</td>\n",
       "      <td>{non_irony}</td>\n",
       "      <td>{non-offensive}</td>\n",
       "      <td>{NOT-HATE}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3859</th>\n",
       "      <td>10/6/2021 16:08:39</td>\n",
       "      <td>10/6/2021 16:10:42</td>\n",
       "      <td>213.127.113.113</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>UghFY3QJ6nmT_ngCoAEC.7-H0Z7--wxd8goqpaPs-bl</td>\n",
       "      <td>20000102</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>No</td>\n",
       "      <td>not political/negative</td>\n",
       "      <td>No</td>\n",
       "      <td>Yes</td>\n",
       "      <td>{film_tv_&amp;_video, news_&amp;_social_concern}</td>\n",
       "      <td>{anticipation}</td>\n",
       "      <td>{neutral, negative}</td>\n",
       "      <td>{irony}</td>\n",
       "      <td>{non-offensive}</td>\n",
       "      <td>{NOT-HATE}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3860</th>\n",
       "      <td>10/15/2021 18:30:04</td>\n",
       "      <td>10/15/2021 18:35:40</td>\n",
       "      <td>213.127.109.191</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>UgyWabsmmnq3zam4DgZ4AaABAg</td>\n",
       "      <td>20000418</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>No</td>\n",
       "      <td>not political/negative</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>{news_&amp;_social_concern}</td>\n",
       "      <td>{anticipation, optimism, trust}</td>\n",
       "      <td>{neutral, positive}</td>\n",
       "      <td>{non_irony}</td>\n",
       "      <td>{non-offensive}</td>\n",
       "      <td>{NOT-HATE}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3861</th>\n",
       "      <td>11/19/2021 17:49:17</td>\n",
       "      <td>11/19/2021 17:51:04</td>\n",
       "      <td>213.127.109.191</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>UgwPOHIDyICm10k0Mvx4AaABAg</td>\n",
       "      <td>20001003</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>Yes</td>\n",
       "      <td>political/negative</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>{news_&amp;_social_concern}</td>\n",
       "      <td>{disgust, anger, fear}</td>\n",
       "      <td>{neutral, negative}</td>\n",
       "      <td>{irony}</td>\n",
       "      <td>{non-offensive}</td>\n",
       "      <td>{NOT-HATE}</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3862 rows × 92 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                StartDate         RecordedDate        IPAddress  Finished  \\\n",
       "0      5/30/2021 13:03:17   5/30/2021 13:04:17     62.194.51.29         1   \n",
       "1     10/11/2021 10:34:05  10/11/2021 10:36:46  213.127.109.191         1   \n",
       "2       9/9/2021 18:49:48    9/9/2021 18:51:32    213.127.110.0         1   \n",
       "3       6/6/2021 16:12:46    6/6/2021 16:16:16   213.127.76.145         1   \n",
       "4      6/13/2021 13:25:49   6/13/2021 13:27:28   213.127.82.232         1   \n",
       "...                   ...                  ...              ...       ...   \n",
       "3857   8/19/2021 14:50:13   8/19/2021 14:54:28     62.194.51.29         1   \n",
       "3858   8/19/2021 15:10:27   8/19/2021 15:12:21     62.194.51.29         1   \n",
       "3859   10/6/2021 16:08:39   10/6/2021 16:10:42  213.127.113.113         1   \n",
       "3860  10/15/2021 18:30:04  10/15/2021 18:35:40  213.127.109.191         1   \n",
       "3861  11/19/2021 17:49:17  11/19/2021 17:51:04  213.127.109.191         1   \n",
       "\n",
       "      Coder                                                 ID   Mark_ID  \\\n",
       "0         6  UgyPHwv8G0cDE6-wEgl4AaABAg.8_0ZjJKSJty8_0kXGkAd2U       119   \n",
       "1         6  Ugx2WXq9UdV8mPPjejJ4AaABAg.8yHCKV0Boe58yYRxEQEF45       282   \n",
       "2         6                                1110578710648890000       372   \n",
       "3         6  UgwUPFScjJ0MCeaP2F54AaABAg.8lvp3fc9Euf8lvvgsUgEgV       769   \n",
       "4         6  UgwWKCWtSJdFvjGHvTp4AaABAg.8kUC5dGrQ2H8kUDRihE2f3      1206   \n",
       "...     ...                                                ...       ...   \n",
       "3857      6                                1152219467579100000  10000695   \n",
       "3858      6                                1085362296472430000  10007008   \n",
       "3859      6        UghFY3QJ6nmT_ngCoAEC.7-H0Z7--wxd8goqpaPs-bl  20000102   \n",
       "3860      6                         UgyWabsmmnq3zam4DgZ4AaABAg  20000418   \n",
       "3861      6                         UgwPOHIDyICm10k0Mvx4AaABAg  20001003   \n",
       "\n",
       "      Genre  topiccode  Platform  ...  rationality_topic_relevance  \\\n",
       "0         0          0         1  ...                           No   \n",
       "1         1          2         1  ...                          Yes   \n",
       "2         2          4         2  ...                          Yes   \n",
       "3         0          0         1  ...                          Yes   \n",
       "4         0          0         1  ...                           No   \n",
       "...     ...        ...       ...  ...                          ...   \n",
       "3857      0          4         2  ...                          Yes   \n",
       "3858      1          4         2  ...                          Yes   \n",
       "3859      0          3         1  ...                           No   \n",
       "3860      2          3         1  ...                           No   \n",
       "3861      0          3         1  ...                          Yes   \n",
       "\n",
       "        political_negativity  rationality_background_info  \\\n",
       "0     not political/negative                           No   \n",
       "1     not political/negative                           No   \n",
       "2         political/negative                           No   \n",
       "3         political/negative                          Yes   \n",
       "4     not political/negative                           No   \n",
       "...                      ...                          ...   \n",
       "3857      political/negative                           No   \n",
       "3858      political/negative                           No   \n",
       "3859  not political/negative                           No   \n",
       "3860  not political/negative                           No   \n",
       "3861      political/negative                          Yes   \n",
       "\n",
       "      rationality_reasoning  \\\n",
       "0                        No   \n",
       "1                       Yes   \n",
       "2                        No   \n",
       "3                       Yes   \n",
       "4                        No   \n",
       "...                     ...   \n",
       "3857                    Yes   \n",
       "3858                     No   \n",
       "3859                    Yes   \n",
       "3860                     No   \n",
       "3861                    Yes   \n",
       "\n",
       "                                                 topics  \\\n",
       "0                                                    {}   \n",
       "1                               {news_&_social_concern}   \n",
       "2                               {news_&_social_concern}   \n",
       "3                               {news_&_social_concern}   \n",
       "4                                {diaries_&_daily_life}   \n",
       "...                                                 ...   \n",
       "3857  {news_&_social_concern, business_&_entrepreneu...   \n",
       "3858                            {news_&_social_concern}   \n",
       "3859           {film_tv_&_video, news_&_social_concern}   \n",
       "3860                            {news_&_social_concern}   \n",
       "3861                            {news_&_social_concern}   \n",
       "\n",
       "                             emotions            sentiment  \\\n",
       "0                {pessimism, sadness}  {neutral, negative}   \n",
       "1                    {disgust, anger}  {neutral, negative}   \n",
       "2      {anticipation, anger, disgust}  {neutral, negative}   \n",
       "3                           {disgust}  {neutral, negative}   \n",
       "4                           {sadness}  {neutral, positive}   \n",
       "...                               ...                  ...   \n",
       "3857                 {disgust, anger}  {neutral, negative}   \n",
       "3858                   {anticipation}            {neutral}   \n",
       "3859                   {anticipation}  {neutral, negative}   \n",
       "3860  {anticipation, optimism, trust}  {neutral, positive}   \n",
       "3861           {disgust, anger, fear}  {neutral, negative}   \n",
       "\n",
       "                   irony                   offensive        hate  \n",
       "0     {non_irony, irony}  {offensive, non-offensive}  {NOT-HATE}  \n",
       "1            {non_irony}  {offensive, non-offensive}  {NOT-HATE}  \n",
       "2            {non_irony}             {non-offensive}  {NOT-HATE}  \n",
       "3                {irony}             {non-offensive}  {NOT-HATE}  \n",
       "4     {non_irony, irony}             {non-offensive}  {NOT-HATE}  \n",
       "...                  ...                         ...         ...  \n",
       "3857         {non_irony}             {non-offensive}      {HATE}  \n",
       "3858         {non_irony}             {non-offensive}  {NOT-HATE}  \n",
       "3859             {irony}             {non-offensive}  {NOT-HATE}  \n",
       "3860         {non_irony}             {non-offensive}  {NOT-HATE}  \n",
       "3861             {irony}             {non-offensive}  {NOT-HATE}  \n",
       "\n",
       "[3862 rows x 92 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for labels, preds in Cardiff_predictions.items():\n",
    "    dataset_w_pred_2 = dataset_w_pred_2.join(preds.rename(labels))\n",
    "\n",
    "dataset_w_pred_2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "501dcaae-5cb7-4e84-9210-74bac16b7df5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                StartDate         RecordedDate        IPAddress  Finished  \\\n",
      "0      5/30/2021 13:03:17   5/30/2021 13:04:17     62.194.51.29         1   \n",
      "1     10/11/2021 10:34:05  10/11/2021 10:36:46  213.127.109.191         1   \n",
      "2       9/9/2021 18:49:48    9/9/2021 18:51:32    213.127.110.0         1   \n",
      "3       6/6/2021 16:12:46    6/6/2021 16:16:16   213.127.76.145         1   \n",
      "4      6/13/2021 13:25:49   6/13/2021 13:27:28   213.127.82.232         1   \n",
      "...                   ...                  ...              ...       ...   \n",
      "3857   8/19/2021 14:50:13   8/19/2021 14:54:28     62.194.51.29         1   \n",
      "3858   8/19/2021 15:10:27   8/19/2021 15:12:21     62.194.51.29         1   \n",
      "3859   10/6/2021 16:08:39   10/6/2021 16:10:42  213.127.113.113         1   \n",
      "3860  10/15/2021 18:30:04  10/15/2021 18:35:40  213.127.109.191         1   \n",
      "3861  11/19/2021 17:49:17  11/19/2021 17:51:04  213.127.109.191         1   \n",
      "\n",
      "      Coder                                                 ID   Mark_ID  \\\n",
      "0         6  UgyPHwv8G0cDE6-wEgl4AaABAg.8_0ZjJKSJty8_0kXGkAd2U       119   \n",
      "1         6  Ugx2WXq9UdV8mPPjejJ4AaABAg.8yHCKV0Boe58yYRxEQEF45       282   \n",
      "2         6                                1110578710648890000       372   \n",
      "3         6  UgwUPFScjJ0MCeaP2F54AaABAg.8lvp3fc9Euf8lvvgsUgEgV       769   \n",
      "4         6  UgwWKCWtSJdFvjGHvTp4AaABAg.8kUC5dGrQ2H8kUDRihE2f3      1206   \n",
      "...     ...                                                ...       ...   \n",
      "3857      6                                1152219467579100000  10000695   \n",
      "3858      6                                1085362296472430000  10007008   \n",
      "3859      6        UghFY3QJ6nmT_ngCoAEC.7-H0Z7--wxd8goqpaPs-bl  20000102   \n",
      "3860      6                         UgyWabsmmnq3zam4DgZ4AaABAg  20000418   \n",
      "3861      6                         UgwPOHIDyICm10k0Mvx4AaABAg  20001003   \n",
      "\n",
      "      Genre  topiccode  Platform  ...  rationality_topic_relevance  \\\n",
      "0         0          0         1  ...                           No   \n",
      "1         1          2         1  ...                          Yes   \n",
      "2         2          4         2  ...                          Yes   \n",
      "3         0          0         1  ...                          Yes   \n",
      "4         0          0         1  ...                           No   \n",
      "...     ...        ...       ...  ...                          ...   \n",
      "3857      0          4         2  ...                          Yes   \n",
      "3858      1          4         2  ...                          Yes   \n",
      "3859      0          3         1  ...                           No   \n",
      "3860      2          3         1  ...                           No   \n",
      "3861      0          3         1  ...                          Yes   \n",
      "\n",
      "        political_negativity  rationality_background_info  \\\n",
      "0     not political/negative                           No   \n",
      "1     not political/negative                           No   \n",
      "2         political/negative                           No   \n",
      "3         political/negative                          Yes   \n",
      "4     not political/negative                           No   \n",
      "...                      ...                          ...   \n",
      "3857      political/negative                           No   \n",
      "3858      political/negative                           No   \n",
      "3859  not political/negative                           No   \n",
      "3860  not political/negative                           No   \n",
      "3861      political/negative                          Yes   \n",
      "\n",
      "      rationality_reasoning  \\\n",
      "0                        No   \n",
      "1                       Yes   \n",
      "2                        No   \n",
      "3                       Yes   \n",
      "4                        No   \n",
      "...                     ...   \n",
      "3857                    Yes   \n",
      "3858                     No   \n",
      "3859                    Yes   \n",
      "3860                     No   \n",
      "3861                    Yes   \n",
      "\n",
      "                                                 topics  \\\n",
      "0                                                 set()   \n",
      "1                             {'news_&_social_concern'}   \n",
      "2                             {'news_&_social_concern'}   \n",
      "3                             {'news_&_social_concern'}   \n",
      "4                              {'diaries_&_daily_life'}   \n",
      "...                                                 ...   \n",
      "3857  {'news_&_social_concern', 'business_&_entrepre...   \n",
      "3858                          {'news_&_social_concern'}   \n",
      "3859       {'film_tv_&_video', 'news_&_social_concern'}   \n",
      "3860                          {'news_&_social_concern'}   \n",
      "3861                          {'news_&_social_concern'}   \n",
      "\n",
      "                                   emotions                sentiment  \\\n",
      "0                  {'pessimism', 'sadness'}  {'neutral', 'negative'}   \n",
      "1                      {'disgust', 'anger'}  {'neutral', 'negative'}   \n",
      "2      {'anticipation', 'anger', 'disgust'}  {'neutral', 'negative'}   \n",
      "3                               {'disgust'}  {'neutral', 'negative'}   \n",
      "4                               {'sadness'}  {'neutral', 'positive'}   \n",
      "...                                     ...                      ...   \n",
      "3857                   {'disgust', 'anger'}  {'neutral', 'negative'}   \n",
      "3858                       {'anticipation'}              {'neutral'}   \n",
      "3859                       {'anticipation'}  {'neutral', 'negative'}   \n",
      "3860  {'anticipation', 'optimism', 'trust'}  {'neutral', 'positive'}   \n",
      "3861           {'disgust', 'anger', 'fear'}  {'neutral', 'negative'}   \n",
      "\n",
      "                       irony                       offensive          hate  \n",
      "0     {'non_irony', 'irony'}  {'offensive', 'non-offensive'}  {'NOT-HATE'}  \n",
      "1              {'non_irony'}  {'offensive', 'non-offensive'}  {'NOT-HATE'}  \n",
      "2              {'non_irony'}               {'non-offensive'}  {'NOT-HATE'}  \n",
      "3                  {'irony'}               {'non-offensive'}  {'NOT-HATE'}  \n",
      "4     {'non_irony', 'irony'}               {'non-offensive'}  {'NOT-HATE'}  \n",
      "...                      ...                             ...           ...  \n",
      "3857           {'non_irony'}               {'non-offensive'}      {'HATE'}  \n",
      "3858           {'non_irony'}               {'non-offensive'}  {'NOT-HATE'}  \n",
      "3859               {'irony'}               {'non-offensive'}  {'NOT-HATE'}  \n",
      "3860           {'non_irony'}               {'non-offensive'}  {'NOT-HATE'}  \n",
      "3861               {'irony'}               {'non-offensive'}  {'NOT-HATE'}  \n",
      "\n",
      "[3862 rows x 92 columns]\n"
     ]
    }
   ],
   "source": [
    "# Select all columns that start with 'cardiff'\n",
    "cardiff_columns = ['sentiment', 'offensive', 'topics', 'emotions', 'irony', 'hate']\n",
    "\n",
    "# Convert each selected column to categorical type\n",
    "for col in cardiff_columns:\n",
    "    dataset_w_pred_2[col] = dataset_w_pred_2[col].astype('str').astype('category')\n",
    "\n",
    "# Display the DataFrame to verify the changes\n",
    "print(dataset_w_pred_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "ba3d8120-10e4-4e30-8525-23c18a8ba10d",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_w_pred_2.to_json(f'{CFG.report_dir}/publicsphere.cardiff_prompt_classify_s.json', orient=\"records\", force_ascii=False, indent=4)\n",
    "dataset_w_pred_2.to_parquet(f'{CFG.report_dir}/publicsphere.cardiff_prompt_classify_s.parquet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "cd27c217-912a-4f99-86aa-3b8879181213",
   "metadata": {},
   "outputs": [],
   "source": [
    "#anonymize:\n",
    "dataset_w_pred_anon = dataset_w_pred_2.drop('commentText', axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "7d2aaff0-3c87-47d3-a3c8-4aebdd717ea8",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_w_pred_anon.to_json('data/publicsphere/publicsphere.cardiff_prompt_classify_anon.json', orient=\"records\", force_ascii=False, indent=4)\n",
    "dataset_w_pred_anon.to_parquet('data/publicsphere/publicsphere.cardiff_prompt_classify_anon.parquet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "21a8ffa6-fea8-40bd-9305-82ec58a6fd7d",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_w_pred_anon = pd.read_parquet('data/publicsphere/publicsphere.cardiff_prompt_classify_anon.parquet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "71387d2a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "hate\n",
       "{'NOT-HATE'}            3532\n",
       "{'HATE'}                 276\n",
       "{'HATE', 'NOT-HATE'}      54\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset_w_pred_anon.loc[:, 'hate'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "id": "e0995dfd",
   "metadata": {},
   "outputs": [],
   "source": [
    "#recode offensive to a dummy with non-offensive as 0 and offensive as 1:\n",
    "dataset_w_pred_anon.loc[:, 'offensive_dum'] = dataset_w_pred_anon.loc[:, 'offensive'].map({\"{'non-offensive'}\": 0, \"{'offensive', 'non-offensive'}\": 1, \"{'offensive'}\": 1})\n",
    "#recode hate into a dummy with non-hate as 0 and hate as 1:\n",
    "dataset_w_pred_anon.loc[:, 'hate_dum'] = dataset_w_pred_anon.loc[:, 'hate'].map({\"{'NOT-HATE'}\": 0, \"{'HATE'}\": 1, \"{'HATE', 'NOT-HATE'}\": 1})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "id": "f2f9ae74",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_w_pred_anon.loc[:, 'cardiff_incivil'] = dataset_w_pred_anon.loc[:, ['offensive_dum', 'hate_dum']].max(axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "id": "74d5a135",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "offensive\n",
      "{'non-offensive'}                 2066\n",
      "{'offensive', 'non-offensive'}    1158\n",
      "{'offensive'}                      638\n",
      "Name: count, dtype: int64\n",
      "offensive_dum\n",
      "0    2066\n",
      "1    1796\n",
      "Name: count, dtype: int64\n",
      "hate\n",
      "{'NOT-HATE'}            3532\n",
      "{'HATE'}                 276\n",
      "{'HATE', 'NOT-HATE'}      54\n",
      "Name: count, dtype: int64\n",
      "hate_dum\n",
      "0    3532\n",
      "1     330\n",
      "Name: count, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "print(dataset_w_pred_anon.loc[:, 'offensive'].value_counts())\n",
    "print(dataset_w_pred_anon.loc[:, 'offensive_dum'].value_counts())\n",
    "print(dataset_w_pred_anon.loc[:, 'hate'].value_counts())\n",
    "print(dataset_w_pred_anon.loc[:, 'hate_dum'].value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "8c681d5a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dimension</th>\n",
       "      <th>labels</th>\n",
       "      <th>measures</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Incivility</td>\n",
       "      <td>[INCIVILITY_DUMMY, HATELIST_FOCUSED_DUMMY]</td>\n",
       "      <td>[cardiff_incivil, hate_dum, offensive_dum]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    dimension                                      labels  \\\n",
       "0  Incivility  [INCIVILITY_DUMMY, HATELIST_FOCUSED_DUMMY]   \n",
       "\n",
       "                                     measures  \n",
       "0  [cardiff_incivil, hate_dum, offensive_dum]  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dimensions = pd.DataFrame([['Incivility',\n",
    "               ['INCIVILITY_DUMMY','HATELIST_FOCUSED_DUMMY'],\n",
    "               ['cardiff_incivil','hate_dum','offensive_dum'\t\n",
    "               ]]]  , \n",
    "            columns = ['dimension','labels','measures'])\n",
    "dimensions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "a179cec3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "int64\n"
     ]
    }
   ],
   "source": [
    "print(dataset_w_pred_anon.hate_dum.dtypes)  # Verify the data types"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "e64d92eb",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\sstolwi\\AppData\\Local\\Temp\\ipykernel_11660\\3664728362.py:26: FutureWarning: The behavior of DataFrame concatenation with empty or all-NA entries is deprecated. In a future version, this will no longer exclude empty or all-NA columns when determining the result dtypes. To retain the old behavior, exclude the relevant entries before the concat operation.\n",
      "  performance = pd.concat([performance, pd.DataFrame([row])], ignore_index=True)\n"
     ]
    }
   ],
   "source": [
    "#calculate performance of cardiff's incivility classification:\n",
    "\n",
    "# Reinitialize the DataFrame\n",
    "performance = pd.DataFrame(columns=['Dimension', 'Label', 'Measures', 'Class', 'Precision', 'Recall', 'F1-score', 'support', 'Accuracy_overall'])\n",
    "\n",
    "classif = ['0', '1', 'macro avg', 'weighted avg']\n",
    "\n",
    "for index, dim in dimensions.iterrows():\n",
    "    for label in dim['labels']:\n",
    "        for measure in dim['measures']:\n",
    "            try:\n",
    "                data = dataset_w_pred_anon[dataset_w_pred_anon[measure].notna()]\n",
    "                classfication = classification_report(data[label], data[measure], output_dict=True)\n",
    "                for c in classif:\n",
    "                    row = {\n",
    "                        'Dimension': dim['dimension'],\n",
    "                        'Label': label,\n",
    "                        'Measures': measure,\n",
    "                        'Class': c,\n",
    "                        'Precision': classfication[c]['precision'],\n",
    "                        'Recall': classfication[c]['recall'],\n",
    "                        'F1-score': classfication[c]['f1-score'],\n",
    "                        'support': classfication[c]['support'],\n",
    "                        'Accuracy_overall': classfication['accuracy']\n",
    "                    }\n",
    "                    performance = pd.concat([performance, pd.DataFrame([row])], ignore_index=True)\n",
    "            except IndexError:\n",
    "                pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "a6f50266",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Dimension</th>\n",
       "      <th>Label</th>\n",
       "      <th>Measures</th>\n",
       "      <th>Class</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Recall</th>\n",
       "      <th>F1-score</th>\n",
       "      <th>support</th>\n",
       "      <th>Accuracy_overall</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Incivility</td>\n",
       "      <td>INCIVILITY_DUMMY</td>\n",
       "      <td>cardiff_incivil</td>\n",
       "      <td>0</td>\n",
       "      <td>0.725886</td>\n",
       "      <td>0.746835</td>\n",
       "      <td>0.736212</td>\n",
       "      <td>1975.0</td>\n",
       "      <td>0.726308</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Incivility</td>\n",
       "      <td>INCIVILITY_DUMMY</td>\n",
       "      <td>cardiff_incivil</td>\n",
       "      <td>1</td>\n",
       "      <td>0.726776</td>\n",
       "      <td>0.704822</td>\n",
       "      <td>0.715631</td>\n",
       "      <td>1887.0</td>\n",
       "      <td>0.726308</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Incivility</td>\n",
       "      <td>INCIVILITY_DUMMY</td>\n",
       "      <td>cardiff_incivil</td>\n",
       "      <td>macro avg</td>\n",
       "      <td>0.726331</td>\n",
       "      <td>0.725829</td>\n",
       "      <td>0.725921</td>\n",
       "      <td>3862.0</td>\n",
       "      <td>0.726308</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Incivility</td>\n",
       "      <td>INCIVILITY_DUMMY</td>\n",
       "      <td>cardiff_incivil</td>\n",
       "      <td>weighted avg</td>\n",
       "      <td>0.726321</td>\n",
       "      <td>0.726308</td>\n",
       "      <td>0.726156</td>\n",
       "      <td>3862.0</td>\n",
       "      <td>0.726308</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Incivility</td>\n",
       "      <td>INCIVILITY_DUMMY</td>\n",
       "      <td>hate_dum</td>\n",
       "      <td>0</td>\n",
       "      <td>0.544734</td>\n",
       "      <td>0.974177</td>\n",
       "      <td>0.698747</td>\n",
       "      <td>1975.0</td>\n",
       "      <td>0.570430</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Incivility</td>\n",
       "      <td>INCIVILITY_DUMMY</td>\n",
       "      <td>hate_dum</td>\n",
       "      <td>1</td>\n",
       "      <td>0.845455</td>\n",
       "      <td>0.147854</td>\n",
       "      <td>0.251691</td>\n",
       "      <td>1887.0</td>\n",
       "      <td>0.570430</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Incivility</td>\n",
       "      <td>INCIVILITY_DUMMY</td>\n",
       "      <td>hate_dum</td>\n",
       "      <td>macro avg</td>\n",
       "      <td>0.695094</td>\n",
       "      <td>0.561015</td>\n",
       "      <td>0.475219</td>\n",
       "      <td>3862.0</td>\n",
       "      <td>0.570430</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Incivility</td>\n",
       "      <td>INCIVILITY_DUMMY</td>\n",
       "      <td>hate_dum</td>\n",
       "      <td>weighted avg</td>\n",
       "      <td>0.691668</td>\n",
       "      <td>0.570430</td>\n",
       "      <td>0.480313</td>\n",
       "      <td>3862.0</td>\n",
       "      <td>0.570430</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Incivility</td>\n",
       "      <td>INCIVILITY_DUMMY</td>\n",
       "      <td>offensive_dum</td>\n",
       "      <td>0</td>\n",
       "      <td>0.724105</td>\n",
       "      <td>0.757468</td>\n",
       "      <td>0.740411</td>\n",
       "      <td>1975.0</td>\n",
       "      <td>0.728379</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Incivility</td>\n",
       "      <td>INCIVILITY_DUMMY</td>\n",
       "      <td>offensive_dum</td>\n",
       "      <td>1</td>\n",
       "      <td>0.733296</td>\n",
       "      <td>0.697933</td>\n",
       "      <td>0.715178</td>\n",
       "      <td>1887.0</td>\n",
       "      <td>0.728379</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Incivility</td>\n",
       "      <td>INCIVILITY_DUMMY</td>\n",
       "      <td>offensive_dum</td>\n",
       "      <td>macro avg</td>\n",
       "      <td>0.728700</td>\n",
       "      <td>0.727701</td>\n",
       "      <td>0.727794</td>\n",
       "      <td>3862.0</td>\n",
       "      <td>0.728379</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Incivility</td>\n",
       "      <td>INCIVILITY_DUMMY</td>\n",
       "      <td>offensive_dum</td>\n",
       "      <td>weighted avg</td>\n",
       "      <td>0.728596</td>\n",
       "      <td>0.728379</td>\n",
       "      <td>0.728082</td>\n",
       "      <td>3862.0</td>\n",
       "      <td>0.728379</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Incivility</td>\n",
       "      <td>HATELIST_FOCUSED_DUMMY</td>\n",
       "      <td>cardiff_incivil</td>\n",
       "      <td>0</td>\n",
       "      <td>0.945866</td>\n",
       "      <td>0.639814</td>\n",
       "      <td>0.763304</td>\n",
       "      <td>3004.0</td>\n",
       "      <td>0.691352</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Incivility</td>\n",
       "      <td>HATELIST_FOCUSED_DUMMY</td>\n",
       "      <td>cardiff_incivil</td>\n",
       "      <td>1</td>\n",
       "      <td>0.408743</td>\n",
       "      <td>0.871795</td>\n",
       "      <td>0.556548</td>\n",
       "      <td>858.0</td>\n",
       "      <td>0.691352</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Incivility</td>\n",
       "      <td>HATELIST_FOCUSED_DUMMY</td>\n",
       "      <td>cardiff_incivil</td>\n",
       "      <td>macro avg</td>\n",
       "      <td>0.677305</td>\n",
       "      <td>0.755804</td>\n",
       "      <td>0.659926</td>\n",
       "      <td>3862.0</td>\n",
       "      <td>0.691352</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Incivility</td>\n",
       "      <td>HATELIST_FOCUSED_DUMMY</td>\n",
       "      <td>cardiff_incivil</td>\n",
       "      <td>weighted avg</td>\n",
       "      <td>0.826536</td>\n",
       "      <td>0.691352</td>\n",
       "      <td>0.717370</td>\n",
       "      <td>3862.0</td>\n",
       "      <td>0.691352</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Incivility</td>\n",
       "      <td>HATELIST_FOCUSED_DUMMY</td>\n",
       "      <td>hate_dum</td>\n",
       "      <td>0</td>\n",
       "      <td>0.816818</td>\n",
       "      <td>0.960386</td>\n",
       "      <td>0.882803</td>\n",
       "      <td>3004.0</td>\n",
       "      <td>0.801657</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Incivility</td>\n",
       "      <td>HATELIST_FOCUSED_DUMMY</td>\n",
       "      <td>hate_dum</td>\n",
       "      <td>1</td>\n",
       "      <td>0.639394</td>\n",
       "      <td>0.245921</td>\n",
       "      <td>0.355219</td>\n",
       "      <td>858.0</td>\n",
       "      <td>0.801657</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Incivility</td>\n",
       "      <td>HATELIST_FOCUSED_DUMMY</td>\n",
       "      <td>hate_dum</td>\n",
       "      <td>macro avg</td>\n",
       "      <td>0.728106</td>\n",
       "      <td>0.603153</td>\n",
       "      <td>0.619011</td>\n",
       "      <td>3862.0</td>\n",
       "      <td>0.801657</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>Incivility</td>\n",
       "      <td>HATELIST_FOCUSED_DUMMY</td>\n",
       "      <td>hate_dum</td>\n",
       "      <td>weighted avg</td>\n",
       "      <td>0.777400</td>\n",
       "      <td>0.801657</td>\n",
       "      <td>0.765592</td>\n",
       "      <td>3862.0</td>\n",
       "      <td>0.801657</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>Incivility</td>\n",
       "      <td>HATELIST_FOCUSED_DUMMY</td>\n",
       "      <td>offensive_dum</td>\n",
       "      <td>0</td>\n",
       "      <td>0.944821</td>\n",
       "      <td>0.649800</td>\n",
       "      <td>0.770020</td>\n",
       "      <td>3004.0</td>\n",
       "      <td>0.698084</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>Incivility</td>\n",
       "      <td>HATELIST_FOCUSED_DUMMY</td>\n",
       "      <td>offensive_dum</td>\n",
       "      <td>1</td>\n",
       "      <td>0.414254</td>\n",
       "      <td>0.867133</td>\n",
       "      <td>0.560663</td>\n",
       "      <td>858.0</td>\n",
       "      <td>0.698084</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>Incivility</td>\n",
       "      <td>HATELIST_FOCUSED_DUMMY</td>\n",
       "      <td>offensive_dum</td>\n",
       "      <td>macro avg</td>\n",
       "      <td>0.679537</td>\n",
       "      <td>0.758467</td>\n",
       "      <td>0.665341</td>\n",
       "      <td>3862.0</td>\n",
       "      <td>0.698084</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>Incivility</td>\n",
       "      <td>HATELIST_FOCUSED_DUMMY</td>\n",
       "      <td>offensive_dum</td>\n",
       "      <td>weighted avg</td>\n",
       "      <td>0.826948</td>\n",
       "      <td>0.698084</td>\n",
       "      <td>0.723508</td>\n",
       "      <td>3862.0</td>\n",
       "      <td>0.698084</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Dimension                   Label         Measures         Class  \\\n",
       "0   Incivility        INCIVILITY_DUMMY  cardiff_incivil             0   \n",
       "1   Incivility        INCIVILITY_DUMMY  cardiff_incivil             1   \n",
       "2   Incivility        INCIVILITY_DUMMY  cardiff_incivil     macro avg   \n",
       "3   Incivility        INCIVILITY_DUMMY  cardiff_incivil  weighted avg   \n",
       "4   Incivility        INCIVILITY_DUMMY         hate_dum             0   \n",
       "5   Incivility        INCIVILITY_DUMMY         hate_dum             1   \n",
       "6   Incivility        INCIVILITY_DUMMY         hate_dum     macro avg   \n",
       "7   Incivility        INCIVILITY_DUMMY         hate_dum  weighted avg   \n",
       "8   Incivility        INCIVILITY_DUMMY    offensive_dum             0   \n",
       "9   Incivility        INCIVILITY_DUMMY    offensive_dum             1   \n",
       "10  Incivility        INCIVILITY_DUMMY    offensive_dum     macro avg   \n",
       "11  Incivility        INCIVILITY_DUMMY    offensive_dum  weighted avg   \n",
       "12  Incivility  HATELIST_FOCUSED_DUMMY  cardiff_incivil             0   \n",
       "13  Incivility  HATELIST_FOCUSED_DUMMY  cardiff_incivil             1   \n",
       "14  Incivility  HATELIST_FOCUSED_DUMMY  cardiff_incivil     macro avg   \n",
       "15  Incivility  HATELIST_FOCUSED_DUMMY  cardiff_incivil  weighted avg   \n",
       "16  Incivility  HATELIST_FOCUSED_DUMMY         hate_dum             0   \n",
       "17  Incivility  HATELIST_FOCUSED_DUMMY         hate_dum             1   \n",
       "18  Incivility  HATELIST_FOCUSED_DUMMY         hate_dum     macro avg   \n",
       "19  Incivility  HATELIST_FOCUSED_DUMMY         hate_dum  weighted avg   \n",
       "20  Incivility  HATELIST_FOCUSED_DUMMY    offensive_dum             0   \n",
       "21  Incivility  HATELIST_FOCUSED_DUMMY    offensive_dum             1   \n",
       "22  Incivility  HATELIST_FOCUSED_DUMMY    offensive_dum     macro avg   \n",
       "23  Incivility  HATELIST_FOCUSED_DUMMY    offensive_dum  weighted avg   \n",
       "\n",
       "    Precision    Recall  F1-score  support  Accuracy_overall  \n",
       "0    0.725886  0.746835  0.736212   1975.0          0.726308  \n",
       "1    0.726776  0.704822  0.715631   1887.0          0.726308  \n",
       "2    0.726331  0.725829  0.725921   3862.0          0.726308  \n",
       "3    0.726321  0.726308  0.726156   3862.0          0.726308  \n",
       "4    0.544734  0.974177  0.698747   1975.0          0.570430  \n",
       "5    0.845455  0.147854  0.251691   1887.0          0.570430  \n",
       "6    0.695094  0.561015  0.475219   3862.0          0.570430  \n",
       "7    0.691668  0.570430  0.480313   3862.0          0.570430  \n",
       "8    0.724105  0.757468  0.740411   1975.0          0.728379  \n",
       "9    0.733296  0.697933  0.715178   1887.0          0.728379  \n",
       "10   0.728700  0.727701  0.727794   3862.0          0.728379  \n",
       "11   0.728596  0.728379  0.728082   3862.0          0.728379  \n",
       "12   0.945866  0.639814  0.763304   3004.0          0.691352  \n",
       "13   0.408743  0.871795  0.556548    858.0          0.691352  \n",
       "14   0.677305  0.755804  0.659926   3862.0          0.691352  \n",
       "15   0.826536  0.691352  0.717370   3862.0          0.691352  \n",
       "16   0.816818  0.960386  0.882803   3004.0          0.801657  \n",
       "17   0.639394  0.245921  0.355219    858.0          0.801657  \n",
       "18   0.728106  0.603153  0.619011   3862.0          0.801657  \n",
       "19   0.777400  0.801657  0.765592   3862.0          0.801657  \n",
       "20   0.944821  0.649800  0.770020   3004.0          0.698084  \n",
       "21   0.414254  0.867133  0.560663    858.0          0.698084  \n",
       "22   0.679537  0.758467  0.665341   3862.0          0.698084  \n",
       "23   0.826948  0.698084  0.723508   3862.0          0.698084  "
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "performance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6d5955e1",
   "metadata": {},
   "outputs": [],
   "source": [
    "#pretty good for incivility and on-par with ML model based on bert-base-uncased, fine-tuned on this dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "52c0e873",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_w_pred_2.to_json(f'{CFG.report_dir}/publicsphere.cardiff_prompt_classify_s.json', orient=\"records\", force_ascii=False, indent=4)\n",
    "dataset_w_pred_2.to_parquet(f'{CFG.report_dir}/publicsphere.cardiff_prompt_classify_s.parquet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "c0498a3a",
   "metadata": {},
   "outputs": [],
   "source": [
    "#anonymize:\n",
    "dataset_w_pred_anon = dataset_w_pred_2.drop('commentText', axis=1)\n",
    "dataset_w_pred_anon.to_json('data/publicsphere/publicsphere.cardiff_prompt_classify_anon.json', orient=\"records\", force_ascii=False, indent=4)\n",
    "dataset_w_pred_anon.to_parquet('data/publicsphere/publicsphere.cardiff_prompt_classify_anon.parquet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "5206ac68",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dimension</th>\n",
       "      <th>labels</th>\n",
       "      <th>measures</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Diversity</td>\n",
       "      <td>[HAS_OPINION_DUMMY, LIBERAL_DUMMY, CONSERVATIV...</td>\n",
       "      <td>[political_conservative_US, political_liberal_...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   dimension                                             labels  \\\n",
       "0  Diversity  [HAS_OPINION_DUMMY, LIBERAL_DUMMY, CONSERVATIV...   \n",
       "\n",
       "                                            measures  \n",
       "0  [political_conservative_US, political_liberal_...  "
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dimensions = pd.DataFrame([['Diversity',\n",
    "               ['HAS_OPINION_DUMMY','LIBERAL_DUMMY','CONSERVATIVE_DUMMY'],\n",
    "               ['political_conservative_US', 'political_liberal_US', 'political_opinion_US'\n",
    "               ]]]  , \n",
    "            columns = ['dimension','labels','measures'])\n",
    "dimensions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "ecdc7347",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\sstolwi\\AppData\\Local\\Temp\\ipykernel_4848\\3664728362.py:26: FutureWarning: The behavior of DataFrame concatenation with empty or all-NA entries is deprecated. In a future version, this will no longer exclude empty or all-NA columns when determining the result dtypes. To retain the old behavior, exclude the relevant entries before the concat operation.\n",
      "  performance = pd.concat([performance, pd.DataFrame([row])], ignore_index=True)\n"
     ]
    }
   ],
   "source": [
    "#calculate performance of cardiff's incivility classification:\n",
    "\n",
    "# Reinitialize the DataFrame\n",
    "performance = pd.DataFrame(columns=['Dimension', 'Label', 'Measures', 'Class', 'Precision', 'Recall', 'F1-score', 'support', 'Accuracy_overall'])\n",
    "\n",
    "classif = ['0', '1', 'macro avg', 'weighted avg']\n",
    "\n",
    "for index, dim in dimensions.iterrows():\n",
    "    for label in dim['labels']:\n",
    "        for measure in dim['measures']:\n",
    "            try:\n",
    "                data = dataset_w_pred_anon[dataset_w_pred_anon[measure].notna()]\n",
    "                classfication = classification_report(data[label], data[measure], output_dict=True)\n",
    "                for c in classif:\n",
    "                    row = {\n",
    "                        'Dimension': dim['dimension'],\n",
    "                        'Label': label,\n",
    "                        'Measures': measure,\n",
    "                        'Class': c,\n",
    "                        'Precision': classfication[c]['precision'],\n",
    "                        'Recall': classfication[c]['recall'],\n",
    "                        'F1-score': classfication[c]['f1-score'],\n",
    "                        'support': classfication[c]['support'],\n",
    "                        'Accuracy_overall': classfication['accuracy']\n",
    "                    }\n",
    "                    performance = pd.concat([performance, pd.DataFrame([row])], ignore_index=True)\n",
    "            except IndexError:\n",
    "                pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "0d0248f2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Dimension</th>\n",
       "      <th>Label</th>\n",
       "      <th>Measures</th>\n",
       "      <th>Class</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Recall</th>\n",
       "      <th>F1-score</th>\n",
       "      <th>support</th>\n",
       "      <th>Accuracy_overall</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Diversity</td>\n",
       "      <td>HAS_OPINION_DUMMY</td>\n",
       "      <td>political_conservative_US</td>\n",
       "      <td>0</td>\n",
       "      <td>0.572076</td>\n",
       "      <td>0.926062</td>\n",
       "      <td>0.707249</td>\n",
       "      <td>1907.0</td>\n",
       "      <td>0.621440</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Diversity</td>\n",
       "      <td>HAS_OPINION_DUMMY</td>\n",
       "      <td>political_conservative_US</td>\n",
       "      <td>1</td>\n",
       "      <td>0.818065</td>\n",
       "      <td>0.324297</td>\n",
       "      <td>0.464469</td>\n",
       "      <td>1955.0</td>\n",
       "      <td>0.621440</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Diversity</td>\n",
       "      <td>HAS_OPINION_DUMMY</td>\n",
       "      <td>political_conservative_US</td>\n",
       "      <td>macro avg</td>\n",
       "      <td>0.695070</td>\n",
       "      <td>0.625179</td>\n",
       "      <td>0.585859</td>\n",
       "      <td>3862.0</td>\n",
       "      <td>0.621440</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Diversity</td>\n",
       "      <td>HAS_OPINION_DUMMY</td>\n",
       "      <td>political_conservative_US</td>\n",
       "      <td>weighted avg</td>\n",
       "      <td>0.696599</td>\n",
       "      <td>0.621440</td>\n",
       "      <td>0.584350</td>\n",
       "      <td>3862.0</td>\n",
       "      <td>0.621440</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Diversity</td>\n",
       "      <td>HAS_OPINION_DUMMY</td>\n",
       "      <td>political_liberal_US</td>\n",
       "      <td>0</td>\n",
       "      <td>0.602440</td>\n",
       "      <td>0.880440</td>\n",
       "      <td>0.715381</td>\n",
       "      <td>1907.0</td>\n",
       "      <td>0.654065</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Diversity</td>\n",
       "      <td>HAS_OPINION_DUMMY</td>\n",
       "      <td>political_liberal_US</td>\n",
       "      <td>1</td>\n",
       "      <td>0.787907</td>\n",
       "      <td>0.433248</td>\n",
       "      <td>0.559076</td>\n",
       "      <td>1955.0</td>\n",
       "      <td>0.654065</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Diversity</td>\n",
       "      <td>HAS_OPINION_DUMMY</td>\n",
       "      <td>political_liberal_US</td>\n",
       "      <td>macro avg</td>\n",
       "      <td>0.695173</td>\n",
       "      <td>0.656844</td>\n",
       "      <td>0.637229</td>\n",
       "      <td>3862.0</td>\n",
       "      <td>0.654065</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Diversity</td>\n",
       "      <td>HAS_OPINION_DUMMY</td>\n",
       "      <td>political_liberal_US</td>\n",
       "      <td>weighted avg</td>\n",
       "      <td>0.696326</td>\n",
       "      <td>0.654065</td>\n",
       "      <td>0.636257</td>\n",
       "      <td>3862.0</td>\n",
       "      <td>0.654065</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Diversity</td>\n",
       "      <td>HAS_OPINION_DUMMY</td>\n",
       "      <td>political_opinion_US</td>\n",
       "      <td>0</td>\n",
       "      <td>0.764414</td>\n",
       "      <td>0.806502</td>\n",
       "      <td>0.784894</td>\n",
       "      <td>1907.0</td>\n",
       "      <td>0.781719</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Diversity</td>\n",
       "      <td>HAS_OPINION_DUMMY</td>\n",
       "      <td>political_opinion_US</td>\n",
       "      <td>1</td>\n",
       "      <td>0.800541</td>\n",
       "      <td>0.757545</td>\n",
       "      <td>0.778449</td>\n",
       "      <td>1955.0</td>\n",
       "      <td>0.781719</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Diversity</td>\n",
       "      <td>HAS_OPINION_DUMMY</td>\n",
       "      <td>political_opinion_US</td>\n",
       "      <td>macro avg</td>\n",
       "      <td>0.782477</td>\n",
       "      <td>0.782024</td>\n",
       "      <td>0.781672</td>\n",
       "      <td>3862.0</td>\n",
       "      <td>0.781719</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Diversity</td>\n",
       "      <td>HAS_OPINION_DUMMY</td>\n",
       "      <td>political_opinion_US</td>\n",
       "      <td>weighted avg</td>\n",
       "      <td>0.782702</td>\n",
       "      <td>0.781719</td>\n",
       "      <td>0.781632</td>\n",
       "      <td>3862.0</td>\n",
       "      <td>0.781719</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Diversity</td>\n",
       "      <td>LIBERAL_DUMMY</td>\n",
       "      <td>political_conservative_US</td>\n",
       "      <td>0</td>\n",
       "      <td>0.759961</td>\n",
       "      <td>0.763423</td>\n",
       "      <td>0.761688</td>\n",
       "      <td>3073.0</td>\n",
       "      <td>0.619886</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Diversity</td>\n",
       "      <td>LIBERAL_DUMMY</td>\n",
       "      <td>political_conservative_US</td>\n",
       "      <td>1</td>\n",
       "      <td>0.061935</td>\n",
       "      <td>0.060837</td>\n",
       "      <td>0.061381</td>\n",
       "      <td>789.0</td>\n",
       "      <td>0.619886</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Diversity</td>\n",
       "      <td>LIBERAL_DUMMY</td>\n",
       "      <td>political_conservative_US</td>\n",
       "      <td>macro avg</td>\n",
       "      <td>0.410948</td>\n",
       "      <td>0.412130</td>\n",
       "      <td>0.411535</td>\n",
       "      <td>3862.0</td>\n",
       "      <td>0.619886</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Diversity</td>\n",
       "      <td>LIBERAL_DUMMY</td>\n",
       "      <td>political_conservative_US</td>\n",
       "      <td>weighted avg</td>\n",
       "      <td>0.617356</td>\n",
       "      <td>0.619886</td>\n",
       "      <td>0.618617</td>\n",
       "      <td>3862.0</td>\n",
       "      <td>0.619886</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Diversity</td>\n",
       "      <td>LIBERAL_DUMMY</td>\n",
       "      <td>political_liberal_US</td>\n",
       "      <td>0</td>\n",
       "      <td>0.933979</td>\n",
       "      <td>0.847055</td>\n",
       "      <td>0.888396</td>\n",
       "      <td>3073.0</td>\n",
       "      <td>0.830658</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Diversity</td>\n",
       "      <td>LIBERAL_DUMMY</td>\n",
       "      <td>political_liberal_US</td>\n",
       "      <td>1</td>\n",
       "      <td>0.562791</td>\n",
       "      <td>0.766793</td>\n",
       "      <td>0.649142</td>\n",
       "      <td>789.0</td>\n",
       "      <td>0.830658</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Diversity</td>\n",
       "      <td>LIBERAL_DUMMY</td>\n",
       "      <td>political_liberal_US</td>\n",
       "      <td>macro avg</td>\n",
       "      <td>0.748385</td>\n",
       "      <td>0.806924</td>\n",
       "      <td>0.768769</td>\n",
       "      <td>3862.0</td>\n",
       "      <td>0.830658</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>Diversity</td>\n",
       "      <td>LIBERAL_DUMMY</td>\n",
       "      <td>political_liberal_US</td>\n",
       "      <td>weighted avg</td>\n",
       "      <td>0.858146</td>\n",
       "      <td>0.830658</td>\n",
       "      <td>0.839517</td>\n",
       "      <td>3862.0</td>\n",
       "      <td>0.830658</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>Diversity</td>\n",
       "      <td>LIBERAL_DUMMY</td>\n",
       "      <td>political_opinion_US</td>\n",
       "      <td>0</td>\n",
       "      <td>0.932406</td>\n",
       "      <td>0.610478</td>\n",
       "      <td>0.737856</td>\n",
       "      <td>3073.0</td>\n",
       "      <td>0.654842</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>Diversity</td>\n",
       "      <td>LIBERAL_DUMMY</td>\n",
       "      <td>political_opinion_US</td>\n",
       "      <td>1</td>\n",
       "      <td>0.352973</td>\n",
       "      <td>0.827630</td>\n",
       "      <td>0.494884</td>\n",
       "      <td>789.0</td>\n",
       "      <td>0.654842</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>Diversity</td>\n",
       "      <td>LIBERAL_DUMMY</td>\n",
       "      <td>political_opinion_US</td>\n",
       "      <td>macro avg</td>\n",
       "      <td>0.642689</td>\n",
       "      <td>0.719054</td>\n",
       "      <td>0.616370</td>\n",
       "      <td>3862.0</td>\n",
       "      <td>0.654842</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>Diversity</td>\n",
       "      <td>LIBERAL_DUMMY</td>\n",
       "      <td>political_opinion_US</td>\n",
       "      <td>weighted avg</td>\n",
       "      <td>0.814028</td>\n",
       "      <td>0.654842</td>\n",
       "      <td>0.688218</td>\n",
       "      <td>3862.0</td>\n",
       "      <td>0.654842</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>Diversity</td>\n",
       "      <td>CONSERVATIVE_DUMMY</td>\n",
       "      <td>political_conservative_US</td>\n",
       "      <td>0</td>\n",
       "      <td>0.950761</td>\n",
       "      <td>0.897279</td>\n",
       "      <td>0.923246</td>\n",
       "      <td>3271.0</td>\n",
       "      <td>0.873641</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>Diversity</td>\n",
       "      <td>CONSERVATIVE_DUMMY</td>\n",
       "      <td>political_conservative_US</td>\n",
       "      <td>1</td>\n",
       "      <td>0.566452</td>\n",
       "      <td>0.742809</td>\n",
       "      <td>0.642753</td>\n",
       "      <td>591.0</td>\n",
       "      <td>0.873641</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>Diversity</td>\n",
       "      <td>CONSERVATIVE_DUMMY</td>\n",
       "      <td>political_conservative_US</td>\n",
       "      <td>macro avg</td>\n",
       "      <td>0.758606</td>\n",
       "      <td>0.820044</td>\n",
       "      <td>0.782999</td>\n",
       "      <td>3862.0</td>\n",
       "      <td>0.873641</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>Diversity</td>\n",
       "      <td>CONSERVATIVE_DUMMY</td>\n",
       "      <td>political_conservative_US</td>\n",
       "      <td>weighted avg</td>\n",
       "      <td>0.891951</td>\n",
       "      <td>0.873641</td>\n",
       "      <td>0.880322</td>\n",
       "      <td>3862.0</td>\n",
       "      <td>0.873641</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>Diversity</td>\n",
       "      <td>CONSERVATIVE_DUMMY</td>\n",
       "      <td>political_liberal_US</td>\n",
       "      <td>0</td>\n",
       "      <td>0.806961</td>\n",
       "      <td>0.687557</td>\n",
       "      <td>0.742489</td>\n",
       "      <td>3271.0</td>\n",
       "      <td>0.596064</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>Diversity</td>\n",
       "      <td>CONSERVATIVE_DUMMY</td>\n",
       "      <td>political_liberal_US</td>\n",
       "      <td>1</td>\n",
       "      <td>0.049302</td>\n",
       "      <td>0.089679</td>\n",
       "      <td>0.063625</td>\n",
       "      <td>591.0</td>\n",
       "      <td>0.596064</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>Diversity</td>\n",
       "      <td>CONSERVATIVE_DUMMY</td>\n",
       "      <td>political_liberal_US</td>\n",
       "      <td>macro avg</td>\n",
       "      <td>0.428132</td>\n",
       "      <td>0.388618</td>\n",
       "      <td>0.403057</td>\n",
       "      <td>3862.0</td>\n",
       "      <td>0.596064</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>Diversity</td>\n",
       "      <td>CONSERVATIVE_DUMMY</td>\n",
       "      <td>political_liberal_US</td>\n",
       "      <td>weighted avg</td>\n",
       "      <td>0.691017</td>\n",
       "      <td>0.596064</td>\n",
       "      <td>0.638603</td>\n",
       "      <td>3862.0</td>\n",
       "      <td>0.596064</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>Diversity</td>\n",
       "      <td>CONSERVATIVE_DUMMY</td>\n",
       "      <td>political_opinion_US</td>\n",
       "      <td>0</td>\n",
       "      <td>0.950795</td>\n",
       "      <td>0.584836</td>\n",
       "      <td>0.724210</td>\n",
       "      <td>3271.0</td>\n",
       "      <td>0.622734</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>Diversity</td>\n",
       "      <td>CONSERVATIVE_DUMMY</td>\n",
       "      <td>political_opinion_US</td>\n",
       "      <td>1</td>\n",
       "      <td>0.265946</td>\n",
       "      <td>0.832487</td>\n",
       "      <td>0.403113</td>\n",
       "      <td>591.0</td>\n",
       "      <td>0.622734</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>Diversity</td>\n",
       "      <td>CONSERVATIVE_DUMMY</td>\n",
       "      <td>political_opinion_US</td>\n",
       "      <td>macro avg</td>\n",
       "      <td>0.608371</td>\n",
       "      <td>0.708662</td>\n",
       "      <td>0.563662</td>\n",
       "      <td>3862.0</td>\n",
       "      <td>0.622734</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>Diversity</td>\n",
       "      <td>CONSERVATIVE_DUMMY</td>\n",
       "      <td>political_opinion_US</td>\n",
       "      <td>weighted avg</td>\n",
       "      <td>0.845993</td>\n",
       "      <td>0.622734</td>\n",
       "      <td>0.675073</td>\n",
       "      <td>3862.0</td>\n",
       "      <td>0.622734</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Dimension               Label                   Measures         Class  \\\n",
       "0   Diversity   HAS_OPINION_DUMMY  political_conservative_US             0   \n",
       "1   Diversity   HAS_OPINION_DUMMY  political_conservative_US             1   \n",
       "2   Diversity   HAS_OPINION_DUMMY  political_conservative_US     macro avg   \n",
       "3   Diversity   HAS_OPINION_DUMMY  political_conservative_US  weighted avg   \n",
       "4   Diversity   HAS_OPINION_DUMMY       political_liberal_US             0   \n",
       "5   Diversity   HAS_OPINION_DUMMY       political_liberal_US             1   \n",
       "6   Diversity   HAS_OPINION_DUMMY       political_liberal_US     macro avg   \n",
       "7   Diversity   HAS_OPINION_DUMMY       political_liberal_US  weighted avg   \n",
       "8   Diversity   HAS_OPINION_DUMMY       political_opinion_US             0   \n",
       "9   Diversity   HAS_OPINION_DUMMY       political_opinion_US             1   \n",
       "10  Diversity   HAS_OPINION_DUMMY       political_opinion_US     macro avg   \n",
       "11  Diversity   HAS_OPINION_DUMMY       political_opinion_US  weighted avg   \n",
       "12  Diversity       LIBERAL_DUMMY  political_conservative_US             0   \n",
       "13  Diversity       LIBERAL_DUMMY  political_conservative_US             1   \n",
       "14  Diversity       LIBERAL_DUMMY  political_conservative_US     macro avg   \n",
       "15  Diversity       LIBERAL_DUMMY  political_conservative_US  weighted avg   \n",
       "16  Diversity       LIBERAL_DUMMY       political_liberal_US             0   \n",
       "17  Diversity       LIBERAL_DUMMY       political_liberal_US             1   \n",
       "18  Diversity       LIBERAL_DUMMY       political_liberal_US     macro avg   \n",
       "19  Diversity       LIBERAL_DUMMY       political_liberal_US  weighted avg   \n",
       "20  Diversity       LIBERAL_DUMMY       political_opinion_US             0   \n",
       "21  Diversity       LIBERAL_DUMMY       political_opinion_US             1   \n",
       "22  Diversity       LIBERAL_DUMMY       political_opinion_US     macro avg   \n",
       "23  Diversity       LIBERAL_DUMMY       political_opinion_US  weighted avg   \n",
       "24  Diversity  CONSERVATIVE_DUMMY  political_conservative_US             0   \n",
       "25  Diversity  CONSERVATIVE_DUMMY  political_conservative_US             1   \n",
       "26  Diversity  CONSERVATIVE_DUMMY  political_conservative_US     macro avg   \n",
       "27  Diversity  CONSERVATIVE_DUMMY  political_conservative_US  weighted avg   \n",
       "28  Diversity  CONSERVATIVE_DUMMY       political_liberal_US             0   \n",
       "29  Diversity  CONSERVATIVE_DUMMY       political_liberal_US             1   \n",
       "30  Diversity  CONSERVATIVE_DUMMY       political_liberal_US     macro avg   \n",
       "31  Diversity  CONSERVATIVE_DUMMY       political_liberal_US  weighted avg   \n",
       "32  Diversity  CONSERVATIVE_DUMMY       political_opinion_US             0   \n",
       "33  Diversity  CONSERVATIVE_DUMMY       political_opinion_US             1   \n",
       "34  Diversity  CONSERVATIVE_DUMMY       political_opinion_US     macro avg   \n",
       "35  Diversity  CONSERVATIVE_DUMMY       political_opinion_US  weighted avg   \n",
       "\n",
       "    Precision    Recall  F1-score  support  Accuracy_overall  \n",
       "0    0.572076  0.926062  0.707249   1907.0          0.621440  \n",
       "1    0.818065  0.324297  0.464469   1955.0          0.621440  \n",
       "2    0.695070  0.625179  0.585859   3862.0          0.621440  \n",
       "3    0.696599  0.621440  0.584350   3862.0          0.621440  \n",
       "4    0.602440  0.880440  0.715381   1907.0          0.654065  \n",
       "5    0.787907  0.433248  0.559076   1955.0          0.654065  \n",
       "6    0.695173  0.656844  0.637229   3862.0          0.654065  \n",
       "7    0.696326  0.654065  0.636257   3862.0          0.654065  \n",
       "8    0.764414  0.806502  0.784894   1907.0          0.781719  \n",
       "9    0.800541  0.757545  0.778449   1955.0          0.781719  \n",
       "10   0.782477  0.782024  0.781672   3862.0          0.781719  \n",
       "11   0.782702  0.781719  0.781632   3862.0          0.781719  \n",
       "12   0.759961  0.763423  0.761688   3073.0          0.619886  \n",
       "13   0.061935  0.060837  0.061381    789.0          0.619886  \n",
       "14   0.410948  0.412130  0.411535   3862.0          0.619886  \n",
       "15   0.617356  0.619886  0.618617   3862.0          0.619886  \n",
       "16   0.933979  0.847055  0.888396   3073.0          0.830658  \n",
       "17   0.562791  0.766793  0.649142    789.0          0.830658  \n",
       "18   0.748385  0.806924  0.768769   3862.0          0.830658  \n",
       "19   0.858146  0.830658  0.839517   3862.0          0.830658  \n",
       "20   0.932406  0.610478  0.737856   3073.0          0.654842  \n",
       "21   0.352973  0.827630  0.494884    789.0          0.654842  \n",
       "22   0.642689  0.719054  0.616370   3862.0          0.654842  \n",
       "23   0.814028  0.654842  0.688218   3862.0          0.654842  \n",
       "24   0.950761  0.897279  0.923246   3271.0          0.873641  \n",
       "25   0.566452  0.742809  0.642753    591.0          0.873641  \n",
       "26   0.758606  0.820044  0.782999   3862.0          0.873641  \n",
       "27   0.891951  0.873641  0.880322   3862.0          0.873641  \n",
       "28   0.806961  0.687557  0.742489   3271.0          0.596064  \n",
       "29   0.049302  0.089679  0.063625    591.0          0.596064  \n",
       "30   0.428132  0.388618  0.403057   3862.0          0.596064  \n",
       "31   0.691017  0.596064  0.638603   3862.0          0.596064  \n",
       "32   0.950795  0.584836  0.724210   3271.0          0.622734  \n",
       "33   0.265946  0.832487  0.403113    591.0          0.622734  \n",
       "34   0.608371  0.708662  0.563662   3862.0          0.622734  \n",
       "35   0.845993  0.622734  0.675073   3862.0          0.622734  "
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "performance  #these are legacy variables, did not set seed or temperature, so results are not reproducible"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a3684c45",
   "metadata": {},
   "outputs": [],
   "source": [
    "#very good performance for all political ideology classifications\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "id": "e4a8a408",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Inter-coder reliablity: classification_report(human as gold, llm as predicted), cohen_kappa(human, llm):\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.93      0.85      0.89      3073\n",
      "           1       0.56      0.77      0.65       789\n",
      "\n",
      "    accuracy                           0.83      3862\n",
      "   macro avg       0.75      0.81      0.77      3862\n",
      "weighted avg       0.86      0.83      0.84      3862\n",
      "\n",
      "cohen_kappa_score(LIBERAL_DUMMY): 0.5409748950370545\n",
      "krippendorf(LIBERAL_DUMMY): 0.5375974087799733\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.95      0.90      0.92      3271\n",
      "           1       0.57      0.74      0.64       591\n",
      "\n",
      "    accuracy                           0.87      3862\n",
      "   macro avg       0.76      0.82      0.78      3862\n",
      "weighted avg       0.89      0.87      0.88      3862\n",
      "\n",
      "cohen_kappa_score(CONSERVATIVE_DUMMY): 0.5676841210411792\n",
      "krippendorf(CONSERVATIVE_DUMMY): 0.566055054744786\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.76      0.81      0.78      1907\n",
      "           1       0.80      0.76      0.78      1955\n",
      "\n",
      "    accuracy                           0.78      3862\n",
      "   macro avg       0.78      0.78      0.78      3862\n",
      "weighted avg       0.78      0.78      0.78      3862\n",
      "\n",
      "cohen_kappa_score(HAS_OPINION_DUMMY): 0.5636661167259271\n",
      "krippendorf(HAS_OPINION_DUMMY): 0.5634000467415223\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.96      0.65      0.78      3165\n",
      "           1       0.36      0.88      0.51       697\n",
      "\n",
      "    accuracy                           0.69      3862\n",
      "   macro avg       0.66      0.77      0.64      3862\n",
      "weighted avg       0.85      0.69      0.73      3862\n",
      "\n",
      "cohen_kappa_score(RATIONALITY_DUMMY): 0.33805429237785745\n",
      "krippendorf(RATIONALITY_DUMMY): 0.2847923827609524\n"
     ]
    }
   ],
   "source": [
    "#these are legacy variables, did not set seed or temperature, so results are not reproducible:\n",
    "llm_human_column_pairs = [\n",
    "    (\"LIBERAL_DUMMY\", \"political_liberal_US\"),\n",
    "    (\"CONSERVATIVE_DUMMY\", \"political_conservative_US\"),\n",
    "    (\"HAS_OPINION_DUMMY\", \"political_opinion_US\"),\n",
    "    (\"RATIONALITY_DUMMY\", \"rationality_prompt_dum\"),\n",
    "   # (\"TopicRelevance\", \"rationality_topic_relevance\"),\n",
    "]\n",
    "print(\"Inter-coder reliablity: classification_report(human as gold, llm as predicted), cohen_kappa(human, llm):\")\n",
    "\n",
    "for human_col, llm_col in llm_human_column_pairs:\n",
    "    subset = dataset_w_pred_anon[[human_col, llm_col]].dropna()\n",
    "    human = subset[human_col].tolist()\n",
    "    llm = subset[llm_col].tolist()\n",
    "    \n",
    "    print(\"---\")\n",
    "    print(f\"{classification_report(human, llm)}\")\n",
    "    print(f\"cohen_kappa_score({human_col}): {cohen_kappa_score(human, llm)}\")\n",
    "    print(f\"krippendorf({human_col}): {krippendorff.alpha(np.array([human, llm]), level_of_measurement=\"nominal\")}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7ee819ae",
   "metadata": {},
   "outputs": [],
   "source": [
    "#although F1-score for rationality is decent, the cohen's kappa is very low, indicating poor agreement between human and LLM classification, probably due to low precision on positive class\n",
    "#performance is on-par than supervised machine learning models in the public sphere paper, but less worse than finetuned bert-base-uncased model (0.72 macro avg precision, recall and f1)\n",
    "#however krippendorf of manual coding for rationality items was in same range as LLM classification, indicating that the task is difficult to classify\n",
    "#apparently mainly the reasoning/argumentation item is difficult to classify (manual K-alpha = 0.3; llm-human K-alpha = 0.05), as the other two items have a higher agreement between human and LLM classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "id": "0842e46b",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_w_pred_anon.to_json('data/publicsphere/publicsphere.cardiff_prompt_classify_anon.json', orient=\"records\", force_ascii=False, indent=4)\n",
    "dataset_w_pred_anon.to_parquet('data/publicsphere/publicsphere.cardiff_prompt_classify_anon.parquet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "13e9d3fb",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_w_pred_anon = pd.read_parquet('data/publicsphere/publicsphere.cardiff_prompt_classify_anon.parquet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "d1c30528",
   "metadata": {},
   "outputs": [],
   "source": [
    "#setup a dummy dataset for incivility to provide to Simon to build a classifyer:\n",
    "#read in the data:\n",
    "dataset_w_pred_2 = pd.read_parquet(f'{CFG.report_dir}/publicsphere.cardiff_prompt_classify_s.parquet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "2a61cec5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                          commentText  INCIVILITY_DUMMY\n",
      "0  This is an English language post 1                 0\n",
      "1  This is an English language post 2                 1\n",
      "2  This is an English language post 3                 0\n",
      "3  This is an English language post 4                 0\n",
      "4  This is an English language post 5                 0\n"
     ]
    }
   ],
   "source": [
    "# Replace the values in the commentText column with sequential text\n",
    "pubsphere_incivility_dummy = dataset_w_pred_2.loc[:5, [\"commentText\", \"INCIVILITY_DUMMY\"]].head()\n",
    "for idx in range(len(pubsphere_incivility_dummy)):\n",
    "    pubsphere_incivility_dummy.at[idx, 'commentText'] = f\"This is an English language post {idx + 1}\"\n",
    "\n",
    "# Display the DataFrame to verify the changes\n",
    "print(pubsphere_incivility_dummy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "52dce0d0",
   "metadata": {},
   "outputs": [],
   "source": [
    "#save data:\n",
    "pubsphere_incivility_dummy.to_parquet('data/publicsphere/publicsphere_incivility_dummy.parquet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "46fbeae7",
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_json(path: str):\n",
    "    with open(path, encoding='utf-8') as fp:\n",
    "        return json.load(fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "a4686763",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Does this comment acknowledge or respond to another user's comment?\n",
      "Instructions: Code Yes (1) if the comment shows agreement or disagreement with a specific user's statement, often signaled by a username or phrases like ‘Yes,’ ‘No,’ or ‘I agree.’ Code No (0) if it lacks a clear acknowledgment or is only an insult.\\n\\nRespond with only the predicted class (0 or 1) of the request.\\n\\nText: {text}\\nClass:\n"
     ]
    }
   ],
   "source": [
    "for label, path in CFG.prompt_classify_files.items():\n",
    "    if label in ['interactivity_acknowledgement_simple']: \n",
    "        template_value = load_json(path).get('template')\n",
    "        print(template_value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "e6dc1b1c",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1it [00:00,  1.03it/s]\n"
     ]
    }
   ],
   "source": [
    "#test Azure api:\n",
    "pubsphereazurerun1 = ['interactivity_acknowledgement_simple']\n",
    "for label, path in CFG.prompt_classify_files.items():\n",
    "    if label in pubsphereazurerun1: \n",
    "        template = load_json(path).get('template')\n",
    "        classes = load_json(path).get('classes')\n",
    "        for index, row in tqdm.tqdm(dataset[\"commentText\"][:1].items()):\n",
    "            response = requests.post(\n",
    "                            url=api_endpoint,\n",
    "                            headers=headers,\n",
    "                            json={\n",
    "                                'model': MODELgpt4o,\n",
    "                                'messages': [\n",
    "                                    {\n",
    "                                        \"role\": \"user\",\n",
    "                                        \"content\": template.format(text=row)\n",
    "                                    }\n",
    "                                ],\n",
    "                                'temperature': temperature_0,  \n",
    "                                'seed': SEED,\n",
    "                                \"max_tokens\": MAX10,\n",
    "                                \"top_p\": TOPP1\n",
    "                            }\n",
    "                        )  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "7d638bb8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'id': 'chatcmpl-AalYNcSXxre3eyg67Xnao3a334562',\n",
       " 'choices': [{'finish_reason': 'stop',\n",
       "   'index': 0,\n",
       "   'message': {'content': '0',\n",
       "    'role': 'assistant',\n",
       "    'tool_calls': None,\n",
       "    'function_call': None}}],\n",
       " 'created': 1733325967,\n",
       " 'model': 'gpt-4o-2024-08-06',\n",
       " 'object': 'chat.completion',\n",
       " 'system_fingerprint': 'fp_04751d0b65',\n",
       " 'usage': {'prompt_tokens': 101,\n",
       "  'completion_tokens': 1,\n",
       "  'total_tokens': 102,\n",
       "  'prompt_tokens_details': {'cached_tokens': 0, 'audio_tokens': 0},\n",
       "  'completion_tokens_details': {'reasoning_tokens': 0,\n",
       "   'audio_tokens': 0,\n",
       "   'accepted_prediction_tokens': 0,\n",
       "   'rejected_prediction_tokens': 0}},\n",
       " 'service_tier': None}"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "response.json()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "48a65938",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1it [00:03,  3.66s/it]\n"
     ]
    }
   ],
   "source": [
    "#test Azure api:\n",
    "pubsphereazurerun1 = ['interactivity_acknowledgement_simple']\n",
    "for label, path in CFG.prompt_classify_files.items():\n",
    "    if label in pubsphereazurerun1: \n",
    "        template = load_json(path).get('template')\n",
    "        classes = load_json(path).get('classes')\n",
    "        for index, row in tqdm.tqdm(dataset[\"commentText\"][:1].items()):\n",
    "            response = requests.post(\n",
    "                            url=api_endpoint,\n",
    "                            headers=headers,\n",
    "                            json={\n",
    "                                'model': MODELgpt4o,\n",
    "                                'messages': [\n",
    "                                    {\n",
    "                                        \"role\": \"user\",\n",
    "                                        \"content\": \"Does this comment acknowledge or respond to another user's comment?\\nInstructions: Code Yes (1) if the comment shows agreement or disagreement with a specific user's statement, often signaled by a username or phrases like ‘Yes,’ ‘No,’ or ‘I agree.’ Code No (0) if it lacks a clear acknowledgment or is only an insult.\\\\n\\\\nRespond with the predicted class (0 or 1) of the request and explain your answer.\\\\n\\\\nText: {text}\\\\nClass:\".format(text=row)\n",
    "                                    }\n",
    "                                ],\n",
    "                                'temperature': temperature_0,  \n",
    "                                'seed': SEED,\n",
    "                                #\"max_tokens\": MAX10,\n",
    "                                \"top_p\": TOPP1\n",
    "                            }\n",
    "                        ) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "18b5b08c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'0\\n\\nExplanation: The comment \"sad\" does not show agreement or disagreement with another user\\'s statement. It lacks any acknowledgment of another user\\'s comment and does not contain any phrases or context that indicate a response to a specific user\\'s statement. It is simply an expression of emotion without reference to another comment.'"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_response = response.json()\n",
    "data_response[\"choices\"][0][\"message\"][\"content\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "aa810863",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3862it [37:36,  1.71it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "      index political_ideology_US\n",
      "0         0               neutral\n",
      "1         1               neutral\n",
      "2         2               liberal\n",
      "3         3               neutral\n",
      "4         4               neutral\n",
      "...     ...                   ...\n",
      "3857   3857               neutral\n",
      "3858   3858               neutral\n",
      "3859   3859               neutral\n",
      "3860   3860               neutral\n",
      "3861   3861               neutral\n",
      "\n",
      "[3862 rows x 2 columns]\n"
     ]
    }
   ],
   "source": [
    "##check simple prompts using GPT4o on Azure:\n",
    "\n",
    "pubsphereazurerun1 = ['interactivity_acknowledgement_simple']   \n",
    "pubsphereazurerun2 = ['rationality_simple2', 'incivility_simple2']\n",
    "pubsphereazurerun3 = ['political_ideology_US']\n",
    "\n",
    "\n",
    "chunked_result: typing.List[pd.DataFrame] = []\n",
    "\n",
    "for label, path in CFG.prompt_classify_files.items():\n",
    "    if label in pubsphereazurerun3: \n",
    "        template = load_json(path).get('template')\n",
    "        classes = load_json(path).get('classes')\n",
    "        for index, row in tqdm.tqdm(dataset[\"commentText\"].items()):\n",
    "            retry_count = 0\n",
    "            max_retries = 5\n",
    "            while retry_count < max_retries:\n",
    "                try: \n",
    "                    response = requests.post(\n",
    "                            url=api_endpoint,\n",
    "                            headers=headers,\n",
    "                            json={\n",
    "                                'model': MODELgpt4o,\n",
    "                                'messages': [\n",
    "                                    {\n",
    "                                        \"role\": \"user\",\n",
    "                                        \"content\": template.format(text=row)\n",
    "                                    }\n",
    "                                ],\n",
    "                                'temperature': temperature_0,  \n",
    "                                'seed': SEED,\n",
    "                                \"max_tokens\": MAX10\n",
    "                            }\n",
    "                        )  \n",
    "\n",
    "                    if response.status_code == 200:\n",
    "                        data_response = response.json()\n",
    "                        chunked_result.append(\n",
    "                        pd.DataFrame(\n",
    "                            data=[[index, classes.get(data_response[\"choices\"][0][\"message\"][\"content\"], None)]],                                \n",
    "                            columns=['index', label]\n",
    "                            )\n",
    "                        )\n",
    "                        break  # Exit the retry loop on success\n",
    "                    elif response.status_code == 429:\n",
    "                        retry_count += 1\n",
    "                        wait_time = 20\n",
    "                        print(f\"Rate limit exceeded. Retrying in {wait_time} seconds...\")\n",
    "                        time.sleep(wait_time)\n",
    "                    else:\n",
    "                        print(f\"Failed to connect to API. Status code: {response.status_code}\")\n",
    "                        print(response.text)\n",
    "                        break\n",
    "                except requests.exceptions.RequestException as e:   \n",
    "                    print(f\"Failed to connect to API: {e}\")\n",
    "                    retry_count += 1\n",
    "                    wait_time = 60\n",
    "                    print(f\"Retrying in {wait_time} seconds...\")\n",
    "                    time.sleep(wait_time)                 \n",
    "                \n",
    "                \n",
    "            # Add a pause of 1 minute after every 70 iterations\n",
    "#            if (index + 1) % 70 == 0:\n",
    "#                print(\"Pausing for 1 minute...\")\n",
    "#                time.sleep(30)\n",
    "\n",
    "classifications3 = pd.concat(chunked_result, ignore_index=True)\n",
    "print(classifications3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5e959679",
   "metadata": {},
   "outputs": [],
   "source": [
    "incivility_gpt4o = classifications.loc[:, [\"index\", \"incivility_simple2\"]].dropna()\n",
    "rationality_gpt4o = classifications.loc[:, [\"index\", \"rationality_simple2\"]].dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "51db4099",
   "metadata": {},
   "outputs": [],
   "source": [
    "classifications3.set_index('index', drop=True, inplace=True)\n",
    "dataset_w_pred_anon = dataset_w_pred_anon.join(classifications3, rsuffix='_gpt4o')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "73b897d1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>StartDate</th>\n",
       "      <th>RecordedDate</th>\n",
       "      <th>IPAddress</th>\n",
       "      <th>Finished</th>\n",
       "      <th>Coder</th>\n",
       "      <th>ID</th>\n",
       "      <th>Mark_ID</th>\n",
       "      <th>Genre</th>\n",
       "      <th>topiccode</th>\n",
       "      <th>Platform</th>\n",
       "      <th>...</th>\n",
       "      <th>interactivity_acknowledgement_simple_gpt4o</th>\n",
       "      <th>incivility_simple2_gpt4o</th>\n",
       "      <th>rationality_simple2_gpt4o</th>\n",
       "      <th>political_ideology_US_gpt4o</th>\n",
       "      <th>incivility_simple2_gpt4o_dum</th>\n",
       "      <th>rationality_simple2_gpt4o_dum</th>\n",
       "      <th>interactivity_acknowledgement_simple_gpt4o_dum</th>\n",
       "      <th>political_conservative_US_gpt4o_dum</th>\n",
       "      <th>political_liberal_US_gpt4o_dum</th>\n",
       "      <th>political_opinion_US_gpt4o_dum</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5/30/2021 13:03:17</td>\n",
       "      <td>5/30/2021 13:04:17</td>\n",
       "      <td>62.194.51.29</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>UgyPHwv8G0cDE6-wEgl4AaABAg.8_0ZjJKSJty8_0kXGkAd2U</td>\n",
       "      <td>119</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>neutral</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>10/11/2021 10:34:05</td>\n",
       "      <td>10/11/2021 10:36:46</td>\n",
       "      <td>213.127.109.191</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>Ugx2WXq9UdV8mPPjejJ4AaABAg.8yHCKV0Boe58yYRxEQEF45</td>\n",
       "      <td>282</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>No</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>neutral</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>9/9/2021 18:49:48</td>\n",
       "      <td>9/9/2021 18:51:32</td>\n",
       "      <td>213.127.110.0</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>1110578710648890000</td>\n",
       "      <td>372</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>liberal</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6/6/2021 16:12:46</td>\n",
       "      <td>6/6/2021 16:16:16</td>\n",
       "      <td>213.127.76.145</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>UgwUPFScjJ0MCeaP2F54AaABAg.8lvp3fc9Euf8lvvgsUgEgV</td>\n",
       "      <td>769</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>neutral</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6/13/2021 13:25:49</td>\n",
       "      <td>6/13/2021 13:27:28</td>\n",
       "      <td>213.127.82.232</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>UgwWKCWtSJdFvjGHvTp4AaABAg.8kUC5dGrQ2H8kUDRihE2f3</td>\n",
       "      <td>1206</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>neutral</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 216 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             StartDate         RecordedDate        IPAddress  Finished  Coder  \\\n",
       "0   5/30/2021 13:03:17   5/30/2021 13:04:17     62.194.51.29         1      6   \n",
       "1  10/11/2021 10:34:05  10/11/2021 10:36:46  213.127.109.191         1      6   \n",
       "2    9/9/2021 18:49:48    9/9/2021 18:51:32    213.127.110.0         1      6   \n",
       "3    6/6/2021 16:12:46    6/6/2021 16:16:16   213.127.76.145         1      6   \n",
       "4   6/13/2021 13:25:49   6/13/2021 13:27:28   213.127.82.232         1      6   \n",
       "\n",
       "                                                  ID  Mark_ID  Genre  \\\n",
       "0  UgyPHwv8G0cDE6-wEgl4AaABAg.8_0ZjJKSJty8_0kXGkAd2U      119      0   \n",
       "1  Ugx2WXq9UdV8mPPjejJ4AaABAg.8yHCKV0Boe58yYRxEQEF45      282      1   \n",
       "2                                1110578710648890000      372      2   \n",
       "3  UgwUPFScjJ0MCeaP2F54AaABAg.8lvp3fc9Euf8lvvgsUgEgV      769      0   \n",
       "4  UgwWKCWtSJdFvjGHvTp4AaABAg.8kUC5dGrQ2H8kUDRihE2f3     1206      0   \n",
       "\n",
       "   topiccode  Platform  ...  interactivity_acknowledgement_simple_gpt4o  \\\n",
       "0          0         1  ...                                          No   \n",
       "1          2         1  ...                                          No   \n",
       "2          4         2  ...                                         Yes   \n",
       "3          0         1  ...                                          No   \n",
       "4          0         1  ...                                          No   \n",
       "\n",
       "  incivility_simple2_gpt4o  rationality_simple2_gpt4o  \\\n",
       "0                       No                         No   \n",
       "1                      Yes                         No   \n",
       "2                       No                         No   \n",
       "3                       No                         No   \n",
       "4                       No                         No   \n",
       "\n",
       "   political_ideology_US_gpt4o  incivility_simple2_gpt4o_dum  \\\n",
       "0                      neutral                             0   \n",
       "1                      neutral                             1   \n",
       "2                      liberal                             0   \n",
       "3                      neutral                             0   \n",
       "4                      neutral                             0   \n",
       "\n",
       "   rationality_simple2_gpt4o_dum  \\\n",
       "0                              0   \n",
       "1                              0   \n",
       "2                              0   \n",
       "3                              0   \n",
       "4                              0   \n",
       "\n",
       "   interactivity_acknowledgement_simple_gpt4o_dum  \\\n",
       "0                                               0   \n",
       "1                                               0   \n",
       "2                                               1   \n",
       "3                                               0   \n",
       "4                                               0   \n",
       "\n",
       "   political_conservative_US_gpt4o_dum  political_liberal_US_gpt4o_dum  \\\n",
       "0                                    0                               0   \n",
       "1                                    0                               0   \n",
       "2                                    0                               1   \n",
       "3                                    0                               0   \n",
       "4                                    0                               0   \n",
       "\n",
       "  political_opinion_US_gpt4o_dum  \n",
       "0                              0  \n",
       "1                              0  \n",
       "2                              1  \n",
       "3                              0  \n",
       "4                              0  \n",
       "\n",
       "[5 rows x 216 columns]"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset_w_pred_anon.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "21275b1e",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_w_pred_anon.to_json('data/publicsphere/publicsphere.cardiff_prompt_classify_anon.json', orient=\"records\", force_ascii=False, indent=4)\n",
    "dataset_w_pred_anon.to_parquet('data/publicsphere/publicsphere.cardiff_prompt_classify_anon.parquet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "073b4afd",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_w_pred_anon.loc[:, 'incivility_simple2_gpt4o_dum'] = dataset_w_pred_anon.loc[:, 'incivility_simple2_gpt4o'].map({\"Yes\": 1, \"No\":0}).fillna(0).astype(int)\n",
    "dataset_w_pred_anon.loc[:, 'rationality_simple2_gpt4o_dum'] = dataset_w_pred_anon.loc[:, 'rationality_simple2_gpt4o'].map({\"Yes\": 1, \"No\":0}).fillna(0).astype(int)\n",
    "dataset_w_pred_anon.loc[:, 'interactivity_acknowledgement_simple_gpt4o_dum'] = dataset_w_pred_anon.loc[:, 'interactivity_acknowledgement_simple_gpt4o'].map({\"Yes\": 1, \"No\":0}).fillna(0).astype(int)\n",
    "\n",
    "dataset_w_pred_anon.loc[:, 'political_conservative_US_gpt4o_dum'] = dataset_w_pred_anon.loc[:, 'political_ideology_US_gpt4o'].map({\"conservative\": 1}).fillna(0).astype(int)\n",
    "dataset_w_pred_anon.loc[:, 'political_liberal_US_gpt4o_dum'] = dataset_w_pred_anon.loc[:, 'political_ideology_US_gpt4o'].map({\"liberal\": 1}).fillna(0).astype(int)\n",
    "dataset_w_pred_anon.loc[:, 'political_opinion_US_gpt4o_dum'] = dataset_w_pred_anon.loc[:, 'political_ideology_US_gpt4o'].map({\"liberal\": 1, 'conservative': 1}).fillna(0).astype(int)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a9223e0f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Inter-coder reliablity: classification_report(human as gold, llm as predicted), cohen_kappa(human, llm):\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.82      0.99      0.90       624\n",
      "           1       0.80      0.11      0.19       149\n",
      "\n",
      "    accuracy                           0.82       773\n",
      "   macro avg       0.81      0.55      0.54       773\n",
      "weighted avg       0.82      0.82      0.76       773\n",
      "\n",
      "cohen_kappa_score(RATIONALITY_DUMMY): 0.15059714301755722\n",
      "krippendorf(RATIONALITY_DUMMY): 0.09044617189413573\n",
      "percent_agreement(RATIONALITY_DUMMY): 82.28%\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.73      0.93      0.82       408\n",
      "           1       0.88      0.61      0.72       365\n",
      "\n",
      "    accuracy                           0.78       773\n",
      "   macro avg       0.81      0.77      0.77       773\n",
      "weighted avg       0.80      0.78      0.77       773\n",
      "\n",
      "cohen_kappa_score(INCIVILITY_DUMMY): 0.5489666185548014\n",
      "krippendorf(INCIVILITY_DUMMY): 0.5395799676898223\n",
      "percent_agreement(INCIVILITY_DUMMY): 77.88%\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.79      0.80      0.79       559\n",
      "           1       0.45      0.43      0.44       214\n",
      "\n",
      "    accuracy                           0.70       773\n",
      "   macro avg       0.62      0.61      0.62       773\n",
      "weighted avg       0.69      0.70      0.70       773\n",
      "\n",
      "cohen_kappa_score(INTERACTIVITY_DUMMY): 0.23290726965844233\n",
      "krippendorf(INTERACTIVITY_DUMMY): 0.2332408632800571\n",
      "percent_agreement(INTERACTIVITY_DUMMY): 69.73%\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.91      0.95      0.93       633\n",
      "           1       0.70      0.58      0.64       140\n",
      "\n",
      "    accuracy                           0.88       773\n",
      "   macro avg       0.81      0.76      0.78       773\n",
      "weighted avg       0.87      0.88      0.87       773\n",
      "\n",
      "cohen_kappa_score(LIBERAL_DUMMY): 0.5640845283934148\n",
      "krippendorf(LIBERAL_DUMMY): 0.5635394359137923\n",
      "percent_agreement(LIBERAL_DUMMY): 87.97%\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.96      0.92      0.94       660\n",
      "           1       0.61      0.75      0.67       113\n",
      "\n",
      "    accuracy                           0.89       773\n",
      "   macro avg       0.78      0.84      0.81       773\n",
      "weighted avg       0.91      0.89      0.90       773\n",
      "\n",
      "cohen_kappa_score(CONSERVATIVE_DUMMY): 0.6120380458067596\n",
      "krippendorf(CONSERVATIVE_DUMMY): 0.6114852432472215\n",
      "percent_agreement(CONSERVATIVE_DUMMY): 89.39%\n"
     ]
    }
   ],
   "source": [
    "#evaluate simple prompts with GPT4o on Azure on TEST SET:\n",
    "\n",
    "llm_human_column_pairs = [\n",
    "    (\"RATIONALITY_DUMMY\", \"rationality_simple2_gpt4o_dum\"),\n",
    "    (\"INCIVILITY_DUMMY\", \"incivility_simple2_gpt4o_dum\"),\n",
    "    (\"INTERACTIVITY_DUMMY\", \"interactivity_acknowledgement_simple_gpt4o_dum\"),\n",
    "    (\"LIBERAL_DUMMY\", \"political_liberal_US_gpt4o_dum\"),\n",
    "    (\"CONSERVATIVE_DUMMY\", \"political_conservative_US_gpt4o_dum\")\n",
    "]\n",
    "\n",
    "print(\"Inter-coder reliablity: classification_report(human as gold, llm as predicted), cohen_kappa(human, llm):\")\n",
    "\n",
    "for human_col, llm_col in llm_human_column_pairs:\n",
    "    subset = dataset_w_pred_anon.loc[dataset_w_pred_anon.split == 'test', [human_col, llm_col]].dropna()\n",
    "    human = subset[human_col].tolist()\n",
    "    llm = subset[llm_col].tolist()\n",
    "\n",
    "    # Calculate percent agreement\n",
    "    agreement = np.sum(np.array(human) == np.array(llm))/len(np.array(human)) * 100\n",
    "\n",
    "    print(\"---\")\n",
    "    print(f\"{classification_report(human, llm)}\")\n",
    "    print(f\"cohen_kappa_score({human_col}): {cohen_kappa_score(human, llm)}\")\n",
    "    print(f\"krippendorf({human_col}): {krippendorff.alpha(np.array([human, llm]), level_of_measurement=\"nominal\")}\")\n",
    "    print(f\"percent_agreement({human_col}): {agreement:.2f}%\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d1dd3f1a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Inter-coder reliablity: classification_report(human as gold, llm as predicted), cohen_kappa(human, llm):\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.84      1.00      0.91      2541\n",
      "           1       0.82      0.10      0.18       548\n",
      "\n",
      "    accuracy                           0.84      3089\n",
      "   macro avg       0.83      0.55      0.55      3089\n",
      "weighted avg       0.83      0.84      0.78      3089\n",
      "\n",
      "cohen_kappa_score(RATIONALITY_DUMMY): 0.14846611270822674\n",
      "krippendorf(RATIONALITY_DUMMY): 0.09135039717563986\n",
      "percent_agreement(RATIONALITY_DUMMY): 83.68%\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.71      0.90      0.79      1567\n",
      "           1       0.86      0.62      0.72      1522\n",
      "\n",
      "    accuracy                           0.76      3089\n",
      "   macro avg       0.79      0.76      0.76      3089\n",
      "weighted avg       0.78      0.76      0.76      3089\n",
      "\n",
      "cohen_kappa_score(INCIVILITY_DUMMY): 0.5240197469407482\n",
      "krippendorf(INCIVILITY_DUMMY): 0.51456155162892\n",
      "percent_agreement(INCIVILITY_DUMMY): 76.30%\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.79      0.78      0.79      2297\n",
      "           1       0.38      0.39      0.39       792\n",
      "\n",
      "    accuracy                           0.68      3089\n",
      "   macro avg       0.59      0.59      0.59      3089\n",
      "weighted avg       0.69      0.68      0.68      3089\n",
      "\n",
      "cohen_kappa_score(INTERACTIVITY_DUMMY): 0.17481793886845698\n",
      "krippendorf(INTERACTIVITY_DUMMY): 0.17490652468086454\n",
      "percent_agreement(INTERACTIVITY_DUMMY): 68.27%\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.90      0.94      0.92      2440\n",
      "           1       0.73      0.61      0.67       649\n",
      "\n",
      "    accuracy                           0.87      3089\n",
      "   macro avg       0.81      0.78      0.79      3089\n",
      "weighted avg       0.87      0.87      0.87      3089\n",
      "\n",
      "cohen_kappa_score(LIBERAL_DUMMY): 0.587268424491223\n",
      "krippendorf(LIBERAL_DUMMY): 0.586629190925517\n",
      "percent_agreement(LIBERAL_DUMMY): 87.08%\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.94      0.91      0.93      2611\n",
      "           1       0.60      0.71      0.65       478\n",
      "\n",
      "    accuracy                           0.88      3089\n",
      "   macro avg       0.77      0.81      0.79      3089\n",
      "weighted avg       0.89      0.88      0.88      3089\n",
      "\n",
      "cohen_kappa_score(CONSERVATIVE_DUMMY): 0.5748164307243147\n",
      "krippendorf(CONSERVATIVE_DUMMY): 0.5742437880862998\n",
      "percent_agreement(CONSERVATIVE_DUMMY): 88.02%\n"
     ]
    }
   ],
   "source": [
    "#evaluate simple prompts with GPT4o on Azure on TRAIN SET:\n",
    "llm_human_column_pairs = [\n",
    "    (\"RATIONALITY_DUMMY\", \"rationality_simple2_gpt4o_dum\"),\n",
    "    (\"INCIVILITY_DUMMY\", \"incivility_simple2_gpt4o_dum\"),\n",
    "    (\"INTERACTIVITY_DUMMY\", \"interactivity_acknowledgement_simple_gpt4o_dum\"),\n",
    "    (\"LIBERAL_DUMMY\", \"political_liberal_US_gpt4o_dum\"),\n",
    "    (\"CONSERVATIVE_DUMMY\", \"political_conservative_US_gpt4o_dum\")\n",
    "]\n",
    "\n",
    "print(\"Inter-coder reliablity: classification_report(human as gold, llm as predicted), cohen_kappa(human, llm):\")\n",
    "\n",
    "for human_col, llm_col in llm_human_column_pairs:\n",
    "    subset = dataset_w_pred_anon.loc[dataset_w_pred_anon.split == 'train', [human_col, llm_col]].dropna()\n",
    "    human = subset[human_col].tolist()\n",
    "    llm = subset[llm_col].tolist()\n",
    "\n",
    "    # Calculate percent agreement\n",
    "    agreement = np.sum(np.array(human) == np.array(llm))/len(np.array(human)) * 100\n",
    "\n",
    "    print(\"---\")\n",
    "    print(f\"{classification_report(human, llm)}\")\n",
    "    print(f\"cohen_kappa_score({human_col}): {cohen_kappa_score(human, llm)}\")\n",
    "    print(f\"krippendorf({human_col}): {krippendorff.alpha(np.array([human, llm]), level_of_measurement=\"nominal\")}\")\n",
    "    print(f\"percent_agreement({human_col}): {agreement:.2f}%\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "959abdb3",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3862it [45:48,  1.41it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "      index interactivity_acknowledgement_simple\n",
      "0         0                                   No\n",
      "1         1                                   No\n",
      "2         2                                  Yes\n",
      "3         3                                   No\n",
      "4         4                                   No\n",
      "...     ...                                  ...\n",
      "3857   3857                                  Yes\n",
      "3858   3858                                  Yes\n",
      "3859   3859                                   No\n",
      "3860   3860                                   No\n",
      "3861   3861                                   No\n",
      "\n",
      "[3862 rows x 2 columns]\n"
     ]
    }
   ],
   "source": [
    "#check effect of removing \"top_p\": TOPP1 from the Azure API call:\n",
    "\n",
    "pubsphereazurerun1 = ['interactivity_acknowledgement_simple']   \n",
    "\n",
    "\n",
    "chunked_result: typing.List[pd.DataFrame] = []\n",
    "\n",
    "for label, path in CFG.prompt_classify_files.items():\n",
    "    if label in pubsphereazurerun1: \n",
    "        template = load_json(path).get('template')\n",
    "        classes = load_json(path).get('classes')\n",
    "        for index, row in tqdm.tqdm(dataset[\"commentText\"].items()):\n",
    "            retry_count = 0\n",
    "            max_retries = 5\n",
    "            while retry_count < max_retries:\n",
    "                try: \n",
    "                    response = requests.post(\n",
    "                            url=api_endpoint,\n",
    "                            headers=headers,\n",
    "                            json={\n",
    "                                'model': MODELgpt4o,\n",
    "                                'messages': [\n",
    "                                    {\n",
    "                                        \"role\": \"user\",\n",
    "                                        \"content\": template.format(text=row)\n",
    "                                    }\n",
    "                                ],\n",
    "                                'temperature': temperature_0,  \n",
    "                                'seed': SEED,\n",
    "                                \"max_tokens\": MAX10\n",
    "                                \n",
    "                            }\n",
    "                        )  \n",
    "\n",
    "                    if response.status_code == 200:\n",
    "                        data_response = response.json()\n",
    "                        chunked_result.append(\n",
    "                        pd.DataFrame(\n",
    "                            data=[[index, classes.get(data_response[\"choices\"][0][\"message\"][\"content\"], None)]],                                \n",
    "                            columns=['index', label]\n",
    "                            )\n",
    "                        )\n",
    "                        break  # Exit the retry loop on success\n",
    "                    elif response.status_code == 429:\n",
    "                        retry_count += 1\n",
    "                        wait_time = 20\n",
    "                        print(f\"Rate limit exceeded. Retrying in {wait_time} seconds...\")\n",
    "                        time.sleep(wait_time)\n",
    "                    else:\n",
    "                        print(f\"Failed to connect to API. Status code: {response.status_code}\")\n",
    "                        print(response.text)\n",
    "                        break\n",
    "                except requests.exceptions.RequestException as e:   \n",
    "                    print(f\"Failed to connect to API: {e}\")\n",
    "                    retry_count += 1\n",
    "                    wait_time = 60\n",
    "                    print(f\"Retrying in {wait_time} seconds...\")\n",
    "                    time.sleep(wait_time)                 \n",
    "                \n",
    "                \n",
    "            # Add a pause of 1 minute after every 70 iterations\n",
    "#            if (index + 1) % 70 == 0:\n",
    "#                print(\"Pausing for 1 minute...\")\n",
    "#                time.sleep(30)\n",
    "\n",
    "classifications3b = pd.concat(chunked_result, ignore_index=True)\n",
    "print(classifications3b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "6aced45f",
   "metadata": {},
   "outputs": [],
   "source": [
    "classifications3b.set_index('index', drop=True, inplace=True)\n",
    "dataset_w_pred_anon = dataset_w_pred_anon.join(classifications3b, rsuffix='_gpt4o_no_top_p')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "04f15d47",
   "metadata": {},
   "outputs": [],
   "source": [
    "#save data:\n",
    "dataset_w_pred_anon.to_json('data/publicsphere/publicsphere.cardiff_prompt_classify_anon.json', orient=\"records\", force_ascii=False, indent=4)\n",
    "dataset_w_pred_anon.to_parquet('data/publicsphere/publicsphere.cardiff_prompt_classify_anon.parquet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "56a8fd57",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>interactivity_acknowledgement_simple_gpt4o_no_top_p</th>\n",
       "      <th>No</th>\n",
       "      <th>Yes</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>interactivity_acknowledgement_simple_gpt4o</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>No</th>\n",
       "      <td>2810</td>\n",
       "      <td>36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Yes</th>\n",
       "      <td>38</td>\n",
       "      <td>978</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "interactivity_acknowledgement_simple_gpt4o_no_top_p    No  Yes\n",
       "interactivity_acknowledgement_simple_gpt4o                    \n",
       "No                                                   2810   36\n",
       "Yes                                                    38  978"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#check differences between the two classifications:\n",
    "pd.crosstab(dataset_w_pred_anon.loc[:, 'interactivity_acknowledgement_simple_gpt4o'], dataset_w_pred_anon.loc[:, 'interactivity_acknowledgement_simple_gpt4o_no_top_p'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "dcab9c09",
   "metadata": {},
   "outputs": [],
   "source": [
    "#doesn't appear to have a strong effect on the classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "492c7d87",
   "metadata": {},
   "outputs": [],
   "source": [
    "#reword interactivity prompt using gpt4o:\n",
    "\n",
    "gpt4o_interactiv_prompt: str = \\\n",
    "    \"\"\"\n",
    "        Does this comment acknowledge or respond to another user's comment?\n",
    "\n",
    "        Instructions:\n",
    "        Classify the comment based on whether it acknowledges or directly responds to another user's statement.\n",
    "\n",
    "        Code \"Yes\" (1): If the comment explicitly engages with another user, such as through agreement, disagreement, or direct reference to their statement. Indicators include:\n",
    "\n",
    "        Mention of usernames.\n",
    "        Use of phrases like \"Yes,\" \"No,\" \"I agree,\" or similar expressions that clearly signal interaction.\n",
    "        Code \"No\" (0): If the comment:\n",
    "\n",
    "        Does not explicitly acknowledge another user's input.\n",
    "        Lacks clear engagement (e.g., is generic, off-topic, or solely insulting).\n",
    "        Respond Format:\n",
    "        Return only the predicted class (0 or 1).\\\\n\\\\nText: {text}\\\\nClass:\n",
    "\t\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "90d8cb50",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3862it [39:08,  1.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "      index interactivity_acknowledgement_simple\n",
      "0         0                                   No\n",
      "1         1                                   No\n",
      "2         2                                  Yes\n",
      "3         3                                   No\n",
      "4         4                                   No\n",
      "...     ...                                  ...\n",
      "3857   3857                                  Yes\n",
      "3858   3858                                  Yes\n",
      "3859   3859                                   No\n",
      "3860   3860                                   No\n",
      "3861   3861                                   No\n",
      "\n",
      "[3862 rows x 2 columns]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "##check reworded prompt using GPT4o on Azure:\n",
    "\n",
    "pubsphereazurerun1 = ['interactivity_acknowledgement_simple']   \n",
    "\n",
    "chunked_result: typing.List[pd.DataFrame] = []\n",
    "\n",
    "for label, path in CFG.prompt_classify_files.items():\n",
    "    if label in pubsphereazurerun1: \n",
    "        template = gpt4o_interactiv_prompt\n",
    "        classes = load_json(path).get('classes')\n",
    "        for index, row in tqdm.tqdm(dataset[\"commentText\"].items()):\n",
    "            retry_count = 0\n",
    "            max_retries = 5\n",
    "            while retry_count < max_retries:\n",
    "                try: \n",
    "                    response = requests.post(\n",
    "                            url=api_endpoint,\n",
    "                            headers=headers,\n",
    "                            json={\n",
    "                                'model': MODELgpt4o,\n",
    "                                'messages': [\n",
    "                                    {\n",
    "                                        \"role\": \"user\",\n",
    "                                        \"content\": template.format(text=row)\n",
    "                                    }\n",
    "                                ],\n",
    "                                'temperature': temperature_0,  \n",
    "                                'seed': SEED,\n",
    "                                \"max_tokens\": MAX10,\n",
    "                                                            }\n",
    "                        )  \n",
    "\n",
    "                    if response.status_code == 200:\n",
    "                        data_response = response.json()\n",
    "                        chunked_result.append(\n",
    "                        pd.DataFrame(\n",
    "                            data=[[index, classes.get(data_response[\"choices\"][0][\"message\"][\"content\"], None)]],                                \n",
    "                            columns=['index', label]\n",
    "                            )\n",
    "                        )\n",
    "                        break  # Exit the retry loop on success\n",
    "                    elif response.status_code == 429:\n",
    "                        retry_count += 1\n",
    "                        wait_time = 20\n",
    "                        print(f\"Rate limit exceeded. Retrying in {wait_time} seconds...\")\n",
    "                        time.sleep(wait_time)\n",
    "                    else:\n",
    "                        print(f\"Failed to connect to API. Status code: {response.status_code}\")\n",
    "                        print(response.text)\n",
    "                        break\n",
    "                except requests.exceptions.RequestException as e:   \n",
    "                    print(f\"Failed to connect to API: {e}\")\n",
    "                    retry_count += 1\n",
    "                    wait_time = 60\n",
    "                    print(f\"Retrying in {wait_time} seconds...\")\n",
    "                    time.sleep(wait_time)                 \n",
    "                \n",
    "                \n",
    "            # Add a pause of 1 minute after every 70 iterations\n",
    "#            if (index + 1) % 70 == 0:\n",
    "#                print(\"Pausing for 1 minute...\")\n",
    "#                time.sleep(30)\n",
    "\n",
    "classifications4 = pd.concat(chunked_result, ignore_index=True)\n",
    "print(classifications4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fa2ef372",
   "metadata": {},
   "outputs": [],
   "source": [
    "classifications4.set_index('index', drop=True, inplace=True)\n",
    "dataset_w_pred_anon = dataset_w_pred_anon.join(classifications4, rsuffix='_gpt4o2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "c9615518",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_w_pred_anon.loc[:, 'interactivity_acknowledgement_gpt4o_2_dum'] = dataset_w_pred_anon.loc[:, 'interactivity_acknowledgement_simple_gpt4o2'].map({\"Yes\": 1, \"No\":0}).fillna(0).astype(int)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "id": "87a60049",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Inter-coder reliablity: classification_report(human as gold, llm as predicted), cohen_kappa(human, llm):\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.79      0.80      0.79       559\n",
      "           1       0.45      0.43      0.44       214\n",
      "\n",
      "    accuracy                           0.70       773\n",
      "   macro avg       0.62      0.61      0.62       773\n",
      "weighted avg       0.69      0.70      0.70       773\n",
      "\n",
      "cohen_kappa_score(INTERACTIVITY_DUMMY): 0.23290726965844233\n",
      "krippendorf(INTERACTIVITY_DUMMY): 0.2332408632800571\n",
      "percent_agreement(INTERACTIVITY_DUMMY): 69.73%\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.77      0.72      0.75       559\n",
      "           1       0.38      0.45      0.41       214\n",
      "\n",
      "    accuracy                           0.65       773\n",
      "   macro avg       0.58      0.59      0.58       773\n",
      "weighted avg       0.67      0.65      0.65       773\n",
      "\n",
      "cohen_kappa_score(INTERACTIVITY_DUMMY): 0.16300997423454466\n",
      "krippendorf(INTERACTIVITY_DUMMY): 0.1608986251843394\n",
      "percent_agreement(INTERACTIVITY_DUMMY): 64.55%\n"
     ]
    }
   ],
   "source": [
    "#evaluate new prompt with GPT4o on Azure on TEST SET:\n",
    "llm_human_column_pairs = [\n",
    "    (\"INTERACTIVITY_DUMMY\", \"interactivity_acknowledgement_simple_gpt4o_dum\"),\n",
    "    (\"INTERACTIVITY_DUMMY\", \"interactivity_acknowledgement_gpt4o_2_dum\"),\n",
    "]\n",
    "\n",
    "print(\"Inter-coder reliablity: classification_report(human as gold, llm as predicted), cohen_kappa(human, llm):\")\n",
    "\n",
    "for human_col, llm_col in llm_human_column_pairs:\n",
    "    subset = dataset_w_pred_anon.loc[dataset_w_pred_anon.split == 'test', [human_col, llm_col]].dropna()\n",
    "    human = subset[human_col].tolist()\n",
    "    llm = subset[llm_col].tolist()\n",
    "\n",
    "    # Calculate percent agreement\n",
    "    agreement = np.sum(np.array(human) == np.array(llm))/len(np.array(human)) * 100\n",
    "\n",
    "    print(\"---\")\n",
    "    print(f\"{classification_report(human, llm)}\")\n",
    "    print(f\"cohen_kappa_score({human_col}): {cohen_kappa_score(human, llm)}\")\n",
    "    print(f\"krippendorf({human_col}): {krippendorff.alpha(np.array([human, llm]), level_of_measurement=\"nominal\")}\")\n",
    "    print(f\"percent_agreement({human_col}): {agreement:.2f}%\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "903c29f9",
   "metadata": {},
   "outputs": [],
   "source": [
    "#clear result, the new prompt doesn't outperform the original short one..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "63f7df96",
   "metadata": {},
   "outputs": [],
   "source": [
    "#potentially gpt4 works better:\n",
    "#https://www.techtarget.com/searchenterpriseai/feature/GPT-4o-vs-GPT-4-How-do-they-compare:\n",
    "# Wat echte gebruikers zeggen, varieert. Op het moment van publicatie is GPT-4o het best beoordeelde model op het crowdsourced LLM-evaluatieplatform LMSYS Chatbot Arena, zowel in het algemeen als in specifieke categorieën zoals coderen en reageren op moeilijke vragen. Maar andere gebruikers noemen GPT-4o \"overhyped\" en melden dat het slechter presteert dan GPT-4 op taken zoals coderen , classificeren en redeneren."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c0e763aa",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3862it [38:43,  1.66it/s]\n",
      "3862it [39:05,  1.65it/s]\n",
      "3862it [38:59,  1.65it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "       index incivility_simple2 political_ideology_US rationality_simple2\n",
      "0          0                 No                   NaN                 NaN\n",
      "1          1                Yes                   NaN                 NaN\n",
      "2          2                 No                   NaN                 NaN\n",
      "3          3                 No                   NaN                 NaN\n",
      "4          4                 No                   NaN                 NaN\n",
      "...      ...                ...                   ...                 ...\n",
      "11581   3857                NaN                   NaN                  No\n",
      "11582   3858                NaN                   NaN                  No\n",
      "11583   3859                NaN                   NaN                  No\n",
      "11584   3860                NaN                   NaN                  No\n",
      "11585   3861                NaN                   NaN                 Yes\n",
      "\n",
      "[11586 rows x 4 columns]\n"
     ]
    }
   ],
   "source": [
    "##check simple prompts using GPT4 *Turbo* on Azure:\n",
    "\n",
    "pubsphereazurerun1 = ['interactivity_acknowledgement_simple']   \n",
    "pubsphereazurerun2 = ['rationality_simple2', 'incivility_simple2', 'political_ideology_US']\n",
    "\n",
    "\n",
    "\n",
    "chunked_result: typing.List[pd.DataFrame] = []\n",
    "\n",
    "for label, path in CFG.prompt_classify_files.items():\n",
    "    if label in pubsphereazurerun2: \n",
    "        template = load_json(path).get('template')\n",
    "        classes = load_json(path).get('classes')\n",
    "        for index, row in tqdm.tqdm(dataset[\"commentText\"].items()):\n",
    "            retry_count = 0\n",
    "            max_retries = 5\n",
    "            while retry_count < max_retries:\n",
    "                try: \n",
    "                    response = requests.post(\n",
    "                            url=api_endpoint,\n",
    "                            headers=headers,\n",
    "                            json={\n",
    "                                'model': MODELgpt4T,\n",
    "                                'messages': [\n",
    "                                    {\n",
    "                                        \"role\": \"user\",\n",
    "                                        \"content\": template.format(text=row)\n",
    "                                    }\n",
    "                                ],\n",
    "                                'temperature': temperature_0,  \n",
    "                                'seed': SEED,\n",
    "                                \"max_tokens\": MAX10                                \n",
    "                            }\n",
    "                        )  \n",
    "\n",
    "                    if response.status_code == 200:\n",
    "                        data_response = response.json()\n",
    "                        chunked_result.append(\n",
    "                        pd.DataFrame(\n",
    "                            data=[[index, classes.get(data_response[\"choices\"][0][\"message\"][\"content\"], None)]],                                \n",
    "                            columns=['index', label]\n",
    "                            )\n",
    "                        )\n",
    "                        break  # Exit the retry loop on success\n",
    "                    elif response.status_code == 429:\n",
    "                        retry_count += 1\n",
    "                        wait_time = 20\n",
    "                        print(f\"Rate limit exceeded. Retrying in {wait_time} seconds...\")\n",
    "                        time.sleep(wait_time)\n",
    "                    else:\n",
    "                        print(f\"Failed to connect to API. Status code: {response.status_code}\")\n",
    "                        print(response.text)\n",
    "                        break\n",
    "                except requests.exceptions.RequestException as e:   \n",
    "                    print(f\"Failed to connect to API: {e}\")\n",
    "                    retry_count += 1\n",
    "                    wait_time = 60\n",
    "                    print(f\"Retrying in {wait_time} seconds...\")\n",
    "                    time.sleep(wait_time)                 \n",
    "                \n",
    "                \n",
    "            # Add a pause of 1 minute after every 70 iterations\n",
    "#            if (index + 1) % 70 == 0:\n",
    "#                print(\"Pausing for 1 minute...\")\n",
    "#                time.sleep(30)\n",
    "\n",
    "classifications6 = pd.concat(chunked_result, ignore_index=True)\n",
    "print(classifications6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "e17a03ba",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'gpt-4-turbo-2024-04-09'"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#check model version:\n",
    "data_response[\"model\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "2a6c85d1",
   "metadata": {},
   "outputs": [],
   "source": [
    "classifications5.set_index('index', drop=True, inplace=True)\n",
    "dataset_w_pred_anon = dataset_w_pred_anon.join(classifications5, rsuffix='_gpt4T')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "c3f6af0f",
   "metadata": {},
   "outputs": [],
   "source": [
    "incivility_gpt4T = classifications6.loc[:, [\"index\", \"incivility_simple2\"]].dropna()\n",
    "rationality_gpt4T = classifications6.loc[:, [\"index\", \"rationality_simple2\"]].dropna()\n",
    "political_ideology_US_gpt4T = classifications6.loc[:, [\"index\", \"political_ideology_US\"]].dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "c7c03eb8",
   "metadata": {},
   "outputs": [],
   "source": [
    "incivility_gpt4T.set_index('index', drop=True, inplace=True)\n",
    "rationality_gpt4T.set_index('index', drop=True, inplace=True)\n",
    "political_ideology_US_gpt4T.set_index('index', drop=True, inplace=True)\n",
    "dataset_w_pred_anon = dataset_w_pred_anon.join(incivility_gpt4T, rsuffix='_gpt4T').join(rationality_gpt4T, rsuffix='_gpt4T').join(political_ideology_US_gpt4T, rsuffix='_gpt4T')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "7f4d6fe9",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_w_pred_anon.loc[:, 'incivility_simple2_gpt4T_dum'] = dataset_w_pred_anon.loc[:, 'incivility_simple2_gpt4T'].map({\"Yes\": 1, \"No\":0}).fillna(0).astype(int)\n",
    "dataset_w_pred_anon.loc[:, 'rationality_simple2_gpt4T_dum'] = dataset_w_pred_anon.loc[:, 'rationality_simple2_gpt4T'].map({\"Yes\": 1, \"No\":0}).fillna(0).astype(int)\n",
    "dataset_w_pred_anon.loc[:, 'interactivity_acknowledgement_simple_gpt4T_dum'] = dataset_w_pred_anon.loc[:, 'interactivity_acknowledgement_simple_gpt4T'].map({\"Yes\": 1, \"No\":0}).fillna(0).astype(int)\n",
    "\n",
    "dataset_w_pred_anon.loc[:, 'political_conservative_US_gpt4T_dum'] = dataset_w_pred_anon.loc[:, 'political_ideology_US_gpt4T'].map({\"conservative\": 1}).fillna(0).astype(int)\n",
    "dataset_w_pred_anon.loc[:, 'political_liberal_US_gpt4T_dum'] = dataset_w_pred_anon.loc[:, 'political_ideology_US_gpt4T'].map({\"liberal\": 1}).fillna(0).astype(int)\n",
    "dataset_w_pred_anon.loc[:, 'political_opinion_US_gpt4T_dum'] = dataset_w_pred_anon.loc[:, 'political_ideology_US_gpt4T'].map({\"liberal\": 1, 'conservative': 1}).fillna(0).astype(int)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "360c8c21",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>StartDate</th>\n",
       "      <th>RecordedDate</th>\n",
       "      <th>IPAddress</th>\n",
       "      <th>Finished</th>\n",
       "      <th>Coder</th>\n",
       "      <th>ID</th>\n",
       "      <th>Mark_ID</th>\n",
       "      <th>Genre</th>\n",
       "      <th>topiccode</th>\n",
       "      <th>Platform</th>\n",
       "      <th>...</th>\n",
       "      <th>interactivity_acknowledgement_simple_gpt4T</th>\n",
       "      <th>incivility_simple2_gpt4T</th>\n",
       "      <th>rationality_simple2_gpt4T</th>\n",
       "      <th>political_ideology_US_gpt4T</th>\n",
       "      <th>incivility_simple2_gpt4T_dum</th>\n",
       "      <th>rationality_simple2_gpt4T_dum</th>\n",
       "      <th>interactivity_acknowledgement_simple_gpt4T_dum</th>\n",
       "      <th>political_conservative_US_gpt4T_dum</th>\n",
       "      <th>political_liberal_US_gpt4T_dum</th>\n",
       "      <th>political_opinion_US_gpt4T_dum</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5/30/2021 13:03:17</td>\n",
       "      <td>5/30/2021 13:04:17</td>\n",
       "      <td>62.194.51.29</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>UgyPHwv8G0cDE6-wEgl4AaABAg.8_0ZjJKSJty8_0kXGkAd2U</td>\n",
       "      <td>119</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>neutral</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>10/11/2021 10:34:05</td>\n",
       "      <td>10/11/2021 10:36:46</td>\n",
       "      <td>213.127.109.191</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>Ugx2WXq9UdV8mPPjejJ4AaABAg.8yHCKV0Boe58yYRxEQEF45</td>\n",
       "      <td>282</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>No</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>neutral</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>9/9/2021 18:49:48</td>\n",
       "      <td>9/9/2021 18:51:32</td>\n",
       "      <td>213.127.110.0</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>1110578710648890000</td>\n",
       "      <td>372</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>liberal</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6/6/2021 16:12:46</td>\n",
       "      <td>6/6/2021 16:16:16</td>\n",
       "      <td>213.127.76.145</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>UgwUPFScjJ0MCeaP2F54AaABAg.8lvp3fc9Euf8lvvgsUgEgV</td>\n",
       "      <td>769</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>neutral</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6/13/2021 13:25:49</td>\n",
       "      <td>6/13/2021 13:27:28</td>\n",
       "      <td>213.127.82.232</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>UgwWKCWtSJdFvjGHvTp4AaABAg.8kUC5dGrQ2H8kUDRihE2f3</td>\n",
       "      <td>1206</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>neutral</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 228 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             StartDate         RecordedDate        IPAddress  Finished  Coder  \\\n",
       "0   5/30/2021 13:03:17   5/30/2021 13:04:17     62.194.51.29         1      6   \n",
       "1  10/11/2021 10:34:05  10/11/2021 10:36:46  213.127.109.191         1      6   \n",
       "2    9/9/2021 18:49:48    9/9/2021 18:51:32    213.127.110.0         1      6   \n",
       "3    6/6/2021 16:12:46    6/6/2021 16:16:16   213.127.76.145         1      6   \n",
       "4   6/13/2021 13:25:49   6/13/2021 13:27:28   213.127.82.232         1      6   \n",
       "\n",
       "                                                  ID  Mark_ID  Genre  \\\n",
       "0  UgyPHwv8G0cDE6-wEgl4AaABAg.8_0ZjJKSJty8_0kXGkAd2U      119      0   \n",
       "1  Ugx2WXq9UdV8mPPjejJ4AaABAg.8yHCKV0Boe58yYRxEQEF45      282      1   \n",
       "2                                1110578710648890000      372      2   \n",
       "3  UgwUPFScjJ0MCeaP2F54AaABAg.8lvp3fc9Euf8lvvgsUgEgV      769      0   \n",
       "4  UgwWKCWtSJdFvjGHvTp4AaABAg.8kUC5dGrQ2H8kUDRihE2f3     1206      0   \n",
       "\n",
       "   topiccode  Platform  ...  interactivity_acknowledgement_simple_gpt4T  \\\n",
       "0          0         1  ...                                          No   \n",
       "1          2         1  ...                                          No   \n",
       "2          4         2  ...                                         Yes   \n",
       "3          0         1  ...                                          No   \n",
       "4          0         1  ...                                          No   \n",
       "\n",
       "  incivility_simple2_gpt4T  rationality_simple2_gpt4T  \\\n",
       "0                       No                         No   \n",
       "1                      Yes                         No   \n",
       "2                       No                         No   \n",
       "3                       No                         No   \n",
       "4                       No                         No   \n",
       "\n",
       "   political_ideology_US_gpt4T  incivility_simple2_gpt4T_dum  \\\n",
       "0                      neutral                             0   \n",
       "1                      neutral                             1   \n",
       "2                      liberal                             0   \n",
       "3                      neutral                             0   \n",
       "4                      neutral                             0   \n",
       "\n",
       "   rationality_simple2_gpt4T_dum  \\\n",
       "0                              0   \n",
       "1                              0   \n",
       "2                              0   \n",
       "3                              0   \n",
       "4                              0   \n",
       "\n",
       "   interactivity_acknowledgement_simple_gpt4T_dum  \\\n",
       "0                                               0   \n",
       "1                                               0   \n",
       "2                                               1   \n",
       "3                                               0   \n",
       "4                                               0   \n",
       "\n",
       "   political_conservative_US_gpt4T_dum  political_liberal_US_gpt4T_dum  \\\n",
       "0                                    0                               0   \n",
       "1                                    0                               0   \n",
       "2                                    0                               1   \n",
       "3                                    0                               0   \n",
       "4                                    0                               0   \n",
       "\n",
       "  political_opinion_US_gpt4T_dum  \n",
       "0                              0  \n",
       "1                              0  \n",
       "2                              1  \n",
       "3                              0  \n",
       "4                              0  \n",
       "\n",
       "[5 rows x 228 columns]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset_w_pred_anon.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "99131fe3",
   "metadata": {},
   "outputs": [],
   "source": [
    "#save data:\n",
    "dataset_w_pred_anon.to_json('data/publicsphere/publicsphere.cardiff_prompt_classify_anon.json', orient=\"records\", force_ascii=False, indent=4)\n",
    "dataset_w_pred_anon.to_parquet('data/publicsphere/publicsphere.cardiff_prompt_classify_anon.parquet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e0b4653f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Inter-coder reliablity: classification_report(human as gold, llm as predicted), cohen_kappa(human, llm):\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.82      0.99      0.90       624\n",
      "           1       0.69      0.12      0.21       149\n",
      "\n",
      "    accuracy                           0.82       773\n",
      "   macro avg       0.76      0.55      0.55       773\n",
      "weighted avg       0.80      0.82      0.77       773\n",
      "\n",
      "cohen_kappa_score(RATIONALITY_DUMMY): 0.1574568522744203\n",
      "krippendorf(RATIONALITY_DUMMY): 0.10490778368240072\n",
      "percent_agreement(RATIONALITY_DUMMY): 82.02%\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.79      0.85      0.82       408\n",
      "           1       0.82      0.75      0.78       365\n",
      "\n",
      "    accuracy                           0.80       773\n",
      "   macro avg       0.80      0.80      0.80       773\n",
      "weighted avg       0.80      0.80      0.80       773\n",
      "\n",
      "cohen_kappa_score(INCIVILITY_DUMMY): 0.6012414318611701\n",
      "krippendorf(INCIVILITY_DUMMY): 0.6009335775603745\n",
      "percent_agreement(INCIVILITY_DUMMY): 80.21%\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.81      0.73      0.77       559\n",
      "           1       0.44      0.55      0.49       214\n",
      "\n",
      "    accuracy                           0.68       773\n",
      "   macro avg       0.62      0.64      0.63       773\n",
      "weighted avg       0.71      0.68      0.69       773\n",
      "\n",
      "cohen_kappa_score(INTERACTIVITY_DUMMY): 0.25862691574414554\n",
      "krippendorf(INTERACTIVITY_DUMMY): 0.25504377617053675\n",
      "percent_agreement(INTERACTIVITY_DUMMY): 68.05%\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.94      0.82      0.88       633\n",
      "           1       0.49      0.78      0.60       140\n",
      "\n",
      "    accuracy                           0.82       773\n",
      "   macro avg       0.72      0.80      0.74       773\n",
      "weighted avg       0.86      0.82      0.83       773\n",
      "\n",
      "cohen_kappa_score(LIBERAL_DUMMY): 0.4910094717114927\n",
      "krippendorf(LIBERAL_DUMMY): 0.48353729092885445\n",
      "percent_agreement(LIBERAL_DUMMY): 81.50%\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.93      0.93      0.93       660\n",
      "           1       0.58      0.58      0.58       113\n",
      "\n",
      "    accuracy                           0.88       773\n",
      "   macro avg       0.75      0.76      0.75       773\n",
      "weighted avg       0.88      0.88      0.88       773\n",
      "\n",
      "cohen_kappa_score(CONSERVATIVE_DUMMY): 0.5094751748415238\n",
      "krippendorf(CONSERVATIVE_DUMMY): 0.5097908240457161\n",
      "percent_agreement(CONSERVATIVE_DUMMY): 87.71%\n"
     ]
    }
   ],
   "source": [
    "#evaluate simple prompts with GPT4 TURBO on Azure on TEST SET:\n",
    "#note temperature=0, top_p=0,9 (default for GPT4 Turbo) and seed=42\n",
    "#model is passed as gpt-4-turbo, is the same results as gpt-4-turbo-2024-04-09 -> see full api output\n",
    "\n",
    "llm_human_column_pairs = [\n",
    "    (\"RATIONALITY_DUMMY\", \"rationality_simple2_gpt4T_dum\"),\n",
    "    (\"INCIVILITY_DUMMY\", \"incivility_simple2_gpt4T_dum\"),\n",
    "    (\"INTERACTIVITY_DUMMY\", \"interactivity_acknowledgement_simple_gpt4T_dum\"),\n",
    "    (\"LIBERAL_DUMMY\", \"political_liberal_US_gpt4T_dum\"),\n",
    "    (\"CONSERVATIVE_DUMMY\", \"political_conservative_US_gpt4T_dum\")\n",
    "]\n",
    "\n",
    "print(\"Inter-coder reliablity: classification_report(human as gold, llm as predicted), cohen_kappa(human, llm):\")\n",
    "\n",
    "for human_col, llm_col in llm_human_column_pairs:\n",
    "    subset = dataset_w_pred_anon.loc[dataset_w_pred_anon.split == 'test', [human_col, llm_col]].dropna()\n",
    "    human = subset[human_col].tolist()\n",
    "    llm = subset[llm_col].tolist()\n",
    "\n",
    "    # Calculate percent agreement\n",
    "    agreement = np.sum(np.array(human) == np.array(llm))/len(np.array(human)) * 100\n",
    "\n",
    "    print(\"---\")\n",
    "    print(f\"{classification_report(human, llm)}\")\n",
    "    print(f\"cohen_kappa_score({human_col}): {cohen_kappa_score(human, llm)}\")\n",
    "    print(f\"krippendorf({human_col}): {krippendorff.alpha(np.array([human, llm]), level_of_measurement=\"nominal\")}\")\n",
    "    print(f\"percent_agreement({human_col}): {agreement:.2f}%\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "62a90468",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Inter-coder reliablity: classification_report(human as gold, llm as predicted), cohen_kappa(human, llm):\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.84      0.99      0.91      2541\n",
      "           1       0.76      0.11      0.20       548\n",
      "\n",
      "    accuracy                           0.84      3089\n",
      "   macro avg       0.80      0.55      0.55      3089\n",
      "weighted avg       0.82      0.84      0.78      3089\n",
      "\n",
      "cohen_kappa_score(RATIONALITY_DUMMY): 0.16050341434222837\n",
      "krippendorf(RATIONALITY_DUMMY): 0.10878712012061165\n",
      "percent_agreement(RATIONALITY_DUMMY): 83.65%\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.78      0.80      0.79      1567\n",
      "           1       0.79      0.76      0.77      1522\n",
      "\n",
      "    accuracy                           0.78      3089\n",
      "   macro avg       0.78      0.78      0.78      3089\n",
      "weighted avg       0.78      0.78      0.78      3089\n",
      "\n",
      "cohen_kappa_score(INCIVILITY_DUMMY): 0.5626621736103696\n",
      "krippendorf(INCIVILITY_DUMMY): 0.5626136547810932\n",
      "percent_agreement(INCIVILITY_DUMMY): 78.15%\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.82      0.73      0.77      2297\n",
      "           1       0.41      0.53      0.46       792\n",
      "\n",
      "    accuracy                           0.68      3089\n",
      "   macro avg       0.61      0.63      0.62      3089\n",
      "weighted avg       0.71      0.68      0.69      3089\n",
      "\n",
      "cohen_kappa_score(INTERACTIVITY_DUMMY): 0.24158375937082466\n",
      "krippendorf(INTERACTIVITY_DUMMY): 0.23620672460894998\n",
      "percent_agreement(INTERACTIVITY_DUMMY): 68.21%\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.92      0.82      0.87      2440\n",
      "           1       0.52      0.75      0.62       649\n",
      "\n",
      "    accuracy                           0.80      3089\n",
      "   macro avg       0.72      0.78      0.74      3089\n",
      "weighted avg       0.84      0.80      0.82      3089\n",
      "\n",
      "cohen_kappa_score(LIBERAL_DUMMY): 0.4908321756455919\n",
      "krippendorf(LIBERAL_DUMMY): 0.4853790805364109\n",
      "percent_agreement(LIBERAL_DUMMY): 80.41%\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.92      0.93      0.93      2611\n",
      "           1       0.61      0.55      0.58       478\n",
      "\n",
      "    accuracy                           0.88      3089\n",
      "   macro avg       0.76      0.74      0.75      3089\n",
      "weighted avg       0.87      0.88      0.87      3089\n",
      "\n",
      "cohen_kappa_score(CONSERVATIVE_DUMMY): 0.5062551299206239\n",
      "krippendorf(CONSERVATIVE_DUMMY): 0.5061627258653825\n",
      "percent_agreement(CONSERVATIVE_DUMMY): 87.54%\n"
     ]
    }
   ],
   "source": [
    "#evaluate simple prompts with GPT4 TURBO on Azure on TRAIN SET:\n",
    "#note temperature=0, top_p=0,9 (default for GPT4 Turbo) and seed=42\n",
    "\n",
    "llm_human_column_pairs = [\n",
    "    (\"RATIONALITY_DUMMY\", \"rationality_simple2_gpt4T_dum\"),\n",
    "    (\"INCIVILITY_DUMMY\", \"incivility_simple2_gpt4T_dum\"),\n",
    "    (\"INTERACTIVITY_DUMMY\", \"interactivity_acknowledgement_simple_gpt4T_dum\"),\n",
    "    (\"LIBERAL_DUMMY\", \"political_liberal_US_gpt4T_dum\"),\n",
    "    (\"CONSERVATIVE_DUMMY\", \"political_conservative_US_gpt4T_dum\")\n",
    "]\n",
    "\n",
    "print(\"Inter-coder reliablity: classification_report(human as gold, llm as predicted), cohen_kappa(human, llm):\")\n",
    "\n",
    "for human_col, llm_col in llm_human_column_pairs:\n",
    "    subset = dataset_w_pred_anon.loc[dataset_w_pred_anon.split == 'train', [human_col, llm_col]].dropna()\n",
    "    human = subset[human_col].tolist()\n",
    "    llm = subset[llm_col].tolist()\n",
    "\n",
    "    # Calculate percent agreement\n",
    "    agreement = np.sum(np.array(human) == np.array(llm))/len(np.array(human)) * 100\n",
    "\n",
    "    print(\"---\")\n",
    "    print(f\"{classification_report(human, llm)}\")\n",
    "    print(f\"cohen_kappa_score({human_col}): {cohen_kappa_score(human, llm)}\")\n",
    "    print(f\"krippendorf({human_col}): {krippendorff.alpha(np.array([human, llm]), level_of_measurement=\"nominal\")}\")\n",
    "    print(f\"percent_agreement({human_col}): {agreement:.2f}%\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "04c98769",
   "metadata": {},
   "source": [
    "#budget spend by classifying pubsphere on GPT4T and GPT4o once: 26.87183725000002"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "21b854d7",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3862it [32:34,  1.98it/s]\n",
      "3862it [33:07,  1.94it/s]\n",
      "3862it [32:04,  2.01it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "       index incivility_simple2 interactivity_acknowledgement_simple  \\\n",
      "0          0                 No                                  NaN   \n",
      "1          1                Yes                                  NaN   \n",
      "2          2                 No                                  NaN   \n",
      "3          3                 No                                  NaN   \n",
      "4          4                 No                                  NaN   \n",
      "...      ...                ...                                  ...   \n",
      "11581   3857                NaN                                  NaN   \n",
      "11582   3858                NaN                                  NaN   \n",
      "11583   3859                NaN                                  NaN   \n",
      "11584   3860                NaN                                  NaN   \n",
      "11585   3861                NaN                                  NaN   \n",
      "\n",
      "      political_ideology_US  \n",
      "0                       NaN  \n",
      "1                       NaN  \n",
      "2                       NaN  \n",
      "3                       NaN  \n",
      "4                       NaN  \n",
      "...                     ...  \n",
      "11581               neutral  \n",
      "11582               neutral  \n",
      "11583               neutral  \n",
      "11584               neutral  \n",
      "11585               neutral  \n",
      "\n",
      "[11586 rows x 4 columns]\n"
     ]
    }
   ],
   "source": [
    "#check effect of adding a system prompt to GPT4o:\n",
    "def load_json(path: str):\n",
    "    with open(path, encoding='utf-8') as fp:\n",
    "        return json.load(fp)\n",
    "    \n",
    "\n",
    "pubsphereazurerun4 = ['rationality_simple2']   \n",
    "pubsphereazurerun5 = ['interactivity_acknowledgement_simple', 'incivility_simple2', 'political_ideology_US']\n",
    "pubsphereazurerun6= ['incivility_simple2', 'political_ideology_US']   \n",
    "\n",
    "\n",
    "chunked_result: typing.List[pd.DataFrame] = []\n",
    "\n",
    "for label, path in CFG.prompt_classify_files.items():\n",
    "    if label in pubsphereazurerun5: \n",
    "        template = load_json(path).get('template')\n",
    "        classes = load_json(path).get('classes')\n",
    "        for index, row in tqdm.tqdm(dataset[\"commentText\"].items()):\n",
    "            retry_count = 0\n",
    "            max_retries = 5\n",
    "            while retry_count < max_retries:\n",
    "                try: \n",
    "                    response = requests.post(\n",
    "                            url=api_endpoint,\n",
    "                            headers=headers,\n",
    "                            json={\n",
    "                                'model': MODELgpt4o,\n",
    "                                'messages': [\n",
    "                                    {\n",
    "                                        \"role\": \"system\",\n",
    "                                        \"content\": template\n",
    "                                    },\n",
    "                                    {\n",
    "                                        \"role\": \"user\",\n",
    "                                        \"content\": template.format(text=row)\n",
    "                                    }\n",
    "                                ],\n",
    "                                'temperature': temperature_0,  \n",
    "                                'seed': SEED,\n",
    "                                \"max_tokens\": MAX10\n",
    "                            }\n",
    "                        )  \n",
    "\n",
    "                    if response.status_code == 200:\n",
    "                        data_response = response.json()\n",
    "                        chunked_result.append(\n",
    "                        pd.DataFrame(\n",
    "                            data=[[index, classes.get(data_response[\"choices\"][0][\"message\"][\"content\"], None)]],                                \n",
    "                            columns=['index', label]\n",
    "                            )\n",
    "                        )\n",
    "                        break  # Exit the retry loop on success\n",
    "                    elif response.status_code == 429:\n",
    "                        retry_count += 1\n",
    "                        wait_time = 20\n",
    "                        print(f\"Rate limit exceeded. Retrying in {wait_time} seconds...\")\n",
    "                        time.sleep(wait_time)\n",
    "                    elif response.status_code == 500:\n",
    "                        retry_count += 1\n",
    "                        wait_time = 20\n",
    "                        print(f\"Failed to connect to API. Status code: {response.status_code}. Retrying in {wait_time} seconds...\")\n",
    "                        time.sleep(wait_time)\n",
    "                    else:\n",
    "                        print(f\"Failed to connect to API. Status code: {response.status_code}\")\n",
    "                        print(response.text)\n",
    "                        break\n",
    "                except requests.exceptions.RequestException as e:   \n",
    "                    print(f\"Failed to connect to API: {e}\")\n",
    "                    retry_count += 1\n",
    "                    wait_time = 60\n",
    "                    print(f\"Retrying in {wait_time} seconds...\")\n",
    "                    time.sleep(wait_time)                 \n",
    "                \n",
    "            #Add a pause of 1 minute after every 70 iterations\n",
    "            #if (index + 1) % 250 == 0:\n",
    "            #    print(\"Pausing for 10 seconds...\")\n",
    "            #    time.sleep(10)\n",
    "\n",
    "classifications3d = pd.concat(chunked_result, ignore_index=True)\n",
    "print(classifications3d) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "ef49bbb5",
   "metadata": {},
   "outputs": [],
   "source": [
    "classifications3c.set_index('index', drop=True, inplace=True)\n",
    "dataset_w_pred_anon = dataset_w_pred_anon.join(classifications3c, rsuffix='_gpt4o_system')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "1af44731",
   "metadata": {},
   "outputs": [],
   "source": [
    "incivility_gpt4o_system = classifications3d.loc[:, [\"index\", \"incivility_simple2\"]].dropna()\n",
    "interactivity_gpt4o_system = classifications3d.loc[:, [\"index\", \"interactivity_acknowledgement_simple\"]].dropna()\n",
    "political_ideology_US_gpt4o_system = classifications3d.loc[:, [\"index\", \"political_ideology_US\"]].dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "da6988cc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>interactivity_acknowledgement_simple</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>index</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3857</th>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3858</th>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3859</th>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3860</th>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3861</th>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3862 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      interactivity_acknowledgement_simple\n",
       "index                                     \n",
       "0                                       No\n",
       "1                                       No\n",
       "2                                      Yes\n",
       "3                                       No\n",
       "4                                       No\n",
       "...                                    ...\n",
       "3857                                   Yes\n",
       "3858                                   Yes\n",
       "3859                                    No\n",
       "3860                                    No\n",
       "3861                                    No\n",
       "\n",
       "[3862 rows x 1 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "341bddc2",
   "metadata": {},
   "outputs": [],
   "source": [
    "incivility_gpt4o_system.set_index('index', drop=True, inplace=True)\n",
    "interactivity_gpt4o_system.set_index('index', drop=True, inplace=True)\n",
    "political_ideology_US_gpt4o_system.set_index('index', drop=True, inplace=True)\n",
    "dataset_w_pred_anon = dataset_w_pred_anon.join(incivility_gpt4o_system, rsuffix='_gpt4o_system').join(interactivity_gpt4o_system, rsuffix='_gpt4o_system').join(political_ideology_US_gpt4o_system, rsuffix='_gpt4o_system')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "4f3056cb",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_w_pred_anon.loc[:, 'incivility_simple2_gpt4o_system_dum'] = dataset_w_pred_anon.loc[:, 'incivility_simple2_gpt4o_system'].map({\"Yes\": 1, \"No\":0}).fillna(0).astype(int)\n",
    "dataset_w_pred_anon.loc[:, 'rationality_simple2_gpt4o_system_dum'] = dataset_w_pred_anon.loc[:, 'rationality_simple2_gpt4o_system'].map({\"Yes\": 1, \"No\":0}).fillna(0).astype(int)\n",
    "dataset_w_pred_anon.loc[:, 'interactivity_acknowledgement_simple_gpt4o_system_dum'] = dataset_w_pred_anon.loc[:, 'interactivity_acknowledgement_simple_gpt4o_system'].map({\"Yes\": 1, \"No\":0}).fillna(0).astype(int)\n",
    "\n",
    "dataset_w_pred_anon.loc[:, 'political_conservative_US_gpt4o_system_dum'] = dataset_w_pred_anon.loc[:, 'political_ideology_US_gpt4o_system'].map({\"conservative\": 1}).fillna(0).astype(int)\n",
    "dataset_w_pred_anon.loc[:, 'political_liberal_US_gpt4o_system_dum'] = dataset_w_pred_anon.loc[:, 'political_ideology_US_gpt4o_system'].map({\"liberal\": 1}).fillna(0).astype(int)\n",
    "dataset_w_pred_anon.loc[:, 'political_opinion_US_gpt4o_system_dum'] = dataset_w_pred_anon.loc[:, 'political_ideology_US_gpt4o_system'].map({\"liberal\": 1, 'conservative': 1}).fillna(0).astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "9646776d",
   "metadata": {},
   "outputs": [],
   "source": [
    "#save data:\n",
    "dataset_w_pred_anon.to_json('data/publicsphere/publicsphere.cardiff_prompt_classify_anon.json', orient=\"records\", force_ascii=False, indent=4)\n",
    "dataset_w_pred_anon.to_parquet('data/publicsphere/publicsphere.cardiff_prompt_classify_anon.parquet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "470b9b6c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['StartDate', 'RecordedDate', 'IPAddress', 'Finished', 'Coder', 'ID',\n",
       "       'Mark_ID', 'Genre', 'topiccode', 'Platform',\n",
       "       ...\n",
       "       'rationality_simple2_gpt4o_system', 'incivility_simple2_gpt4o_system',\n",
       "       'interactivity_acknowledgement_simple_gpt4o_system',\n",
       "       'political_ideology_US_gpt4o_system',\n",
       "       'incivility_simple2_gpt4o_system_dum',\n",
       "       'rationality_simple2_gpt4o_system_dum',\n",
       "       'interactivity_acknowledgement_simple_gpt4o_system_dum',\n",
       "       'political_conservative_US_gpt4o_system_dum',\n",
       "       'political_liberal_US_gpt4o_system_dum',\n",
       "       'political_opinion_US_gpt4o_system_dum'],\n",
       "      dtype='object', length=239)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset_w_pred_anon.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "3e1ebb65",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>rationality_simple2_gpt4o_system</th>\n",
       "      <th>No</th>\n",
       "      <th>Yes</th>\n",
       "      <th>NaN</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rationality_simple2_gpt4o</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>No</th>\n",
       "      <td>3697</td>\n",
       "      <td>73</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Yes</th>\n",
       "      <td>2</td>\n",
       "      <td>86</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "rationality_simple2_gpt4o_system    No  Yes  NaN\n",
       "rationality_simple2_gpt4o                       \n",
       "No                                3697   73    4\n",
       "Yes                                  2   86    0"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#check differences between the two classifications:\n",
    "pd.crosstab(dataset_w_pred_anon.loc[:, 'rationality_simple2_gpt4o'], dataset_w_pred_anon.loc[:, 'rationality_simple2_gpt4o_system'], dropna=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "f2c96bb4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Inter-coder reliablity: classification_report(human as gold, llm as predicted), cohen_kappa(human, llm):\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.85      0.99      0.91      2541\n",
      "           1       0.80      0.19      0.30       548\n",
      "\n",
      "    accuracy                           0.85      3089\n",
      "   macro avg       0.82      0.59      0.61      3089\n",
      "weighted avg       0.84      0.85      0.81      3089\n",
      "\n",
      "cohen_kappa_score(RATIONALITY_DUMMY): 0.2514882877554834\n",
      "krippendorf(RATIONALITY_DUMMY): 0.21611506520490675\n",
      "percent_agreement(RATIONALITY_DUMMY): 84.72%\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.68      0.90      0.78      1567\n",
      "           1       0.85      0.57      0.68      1522\n",
      "\n",
      "    accuracy                           0.74      3089\n",
      "   macro avg       0.77      0.74      0.73      3089\n",
      "weighted avg       0.77      0.74      0.73      3089\n",
      "\n",
      "cohen_kappa_score(INCIVILITY_DUMMY): 0.47486824956083185\n",
      "krippendorf(INCIVILITY_DUMMY): 0.46022346495751765\n",
      "percent_agreement(INCIVILITY_DUMMY): 73.88%\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.81      0.77      0.79      2297\n",
      "           1       0.42      0.48      0.44       792\n",
      "\n",
      "    accuracy                           0.69      3089\n",
      "   macro avg       0.61      0.62      0.62      3089\n",
      "weighted avg       0.71      0.69      0.70      3089\n",
      "\n",
      "cohen_kappa_score(INTERACTIVITY_DUMMY): 0.23483705886912132\n",
      "krippendorf(INTERACTIVITY_DUMMY): 0.2335158694680376\n",
      "percent_agreement(INTERACTIVITY_DUMMY): 69.38%\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.91      0.93      0.92      2440\n",
      "           1       0.71      0.63      0.67       649\n",
      "\n",
      "    accuracy                           0.87      3089\n",
      "   macro avg       0.81      0.78      0.79      3089\n",
      "weighted avg       0.86      0.87      0.87      3089\n",
      "\n",
      "cohen_kappa_score(LIBERAL_DUMMY): 0.5892406213217966\n",
      "krippendorf(LIBERAL_DUMMY): 0.5889564286975519\n",
      "percent_agreement(LIBERAL_DUMMY): 86.92%\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.94      0.93      0.94      2611\n",
      "           1       0.64      0.70      0.67       478\n",
      "\n",
      "    accuracy                           0.89      3089\n",
      "   macro avg       0.79      0.82      0.80      3089\n",
      "weighted avg       0.90      0.89      0.90      3089\n",
      "\n",
      "cohen_kappa_score(CONSERVATIVE_DUMMY): 0.6096725641317935\n",
      "krippendorf(CONSERVATIVE_DUMMY): 0.6095962732379029\n",
      "percent_agreement(CONSERVATIVE_DUMMY): 89.41%\n"
     ]
    }
   ],
   "source": [
    "#Notewithstanding some NAs due to connection issues, the system prompt does not seem to have a strong effect on the classifications, quickly check the effect on the training set:\n",
    "llm_human_column_pairs = [\n",
    "    (\"RATIONALITY_DUMMY\", \"rationality_simple2_gpt4o_system_dum\"),\n",
    "    (\"INCIVILITY_DUMMY\", \"incivility_simple2_gpt4o_system_dum\"),\n",
    "    (\"INTERACTIVITY_DUMMY\", \"interactivity_acknowledgement_simple_gpt4o_system_dum\"),\n",
    "    (\"LIBERAL_DUMMY\", \"political_liberal_US_gpt4o_system_dum\"),\n",
    "    (\"CONSERVATIVE_DUMMY\", \"political_conservative_US_gpt4o_system_dum\")  \n",
    "]\n",
    "\n",
    "print(\"Inter-coder reliablity: classification_report(human as gold, llm as predicted), cohen_kappa(human, llm):\")\n",
    "\n",
    "for human_col, llm_col in llm_human_column_pairs:\n",
    "    subset = dataset_w_pred_anon.loc[dataset_w_pred_anon.split == 'train', [human_col, llm_col]].dropna()\n",
    "    human = subset[human_col].tolist()\n",
    "    llm = subset[llm_col].tolist()\n",
    "\n",
    "    # Calculate percent agreement\n",
    "    agreement = np.sum(np.array(human) == np.array(llm))/len(np.array(human)) * 100\n",
    "\n",
    "    print(\"---\")\n",
    "    print(f\"{classification_report(human, llm)}\")\n",
    "    print(f\"cohen_kappa_score({human_col}): {cohen_kappa_score(human, llm)}\")\n",
    "    print(f\"krippendorf({human_col}): {krippendorff.alpha(np.array([human, llm]), level_of_measurement=\"nominal\")}\")\n",
    "    print(f\"percent_agreement({human_col}): {agreement:.2f}%\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c9a25ad3",
   "metadata": {},
   "outputs": [],
   "source": [
    "#adding system prompt did improve performance for gpt4o on rationality, but not enough to beat Llama3.1_70b\n",
    "#for other variables the large number of API errors makes this analysis unfeasable "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "40457ce1",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "15it [00:06,  2.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "30it [00:36,  1.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "45it [01:27,  1.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "60it [01:55,  2.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "75it [02:26,  1.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "90it [03:17,  1.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "105it [04:09,  1.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "133it [05:34,  1.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "148it [06:03,  1.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "171it [07:24,  1.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "197it [08:03,  2.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "212it [09:17,  1.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "227it [09:44,  2.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "242it [10:15,  1.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "257it [11:06,  1.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "272it [11:35,  1.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "287it [12:27,  1.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "305it [13:00,  1.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "320it [14:14,  1.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "335it [14:41,  1.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "350it [15:11,  1.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "365it [16:25,  1.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "380it [16:57,  1.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "395it [17:27,  1.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "410it [17:55,  1.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "425it [18:25,  1.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "440it [19:16,  1.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "455it [19:44,  1.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "473it [20:19,  2.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "488it [20:46,  1.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "503it [21:16,  1.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "518it [22:08,  1.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "548it [22:58,  1.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "574it [24:23,  1.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "589it [24:52,  1.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "604it [25:21,  2.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "619it [26:13,  1.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "634it [26:41,  2.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "649it [27:10,  1.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "664it [28:25,  1.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "679it [28:53,  1.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "694it [29:24,  1.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "709it [30:14,  1.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "724it [30:41,  2.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "739it [31:10,  1.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "754it [32:25,  1.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "769it [32:53,  1.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "786it [33:27,  1.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "801it [33:54,  1.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "816it [34:24,  1.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "831it [35:16,  1.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "846it [35:43,  1.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "861it [36:13,  2.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "876it [37:26,  1.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "906it [38:06,  2.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "921it [39:19,  1.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "936it [39:48,  2.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "951it [40:17,  1.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "981it [41:22,  1.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "996it [41:50,  1.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1011it [42:20,  2.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1026it [43:11,  1.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1041it [43:40,  1.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1056it [44:09,  1.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1071it [45:23,  1.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1086it [45:50,  2.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1101it [46:19,  2.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1116it [47:11,  1.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1131it [47:39,  1.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1146it [48:08,  2.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1161it [49:20,  1.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1176it [49:50,  2.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1191it [50:19,  2.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1206it [51:10,  1.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1221it [51:38,  1.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1236it [52:07,  2.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1251it [53:22,  1.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1266it [53:50,  1.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1281it [54:21,  1.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1296it [55:12,  1.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1311it [55:41,  1.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1326it [56:10,  1.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1341it [57:22,  1.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1356it [57:50,  1.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1371it [58:20,  1.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1386it [59:11,  1.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1401it [59:39,  2.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1416it [1:00:08,  2.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1431it [1:01:21,  1.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1446it [1:01:49,  1.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1461it [1:02:18,  1.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1476it [1:03:08,  2.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1492it [1:03:39,  1.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1507it [1:04:09,  1.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1522it [1:04:39,  2.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1537it [1:05:10,  1.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1552it [1:05:37,  2.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1567it [1:06:07,  1.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1582it [1:07:20,  1.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1597it [1:07:47,  1.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1612it [1:08:16,  1.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1627it [1:09:08,  1.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1642it [1:09:35,  1.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1657it [1:10:26,  1.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1672it [1:10:54,  1.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1687it [1:11:23,  2.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1702it [1:12:15,  1.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1717it [1:12:42,  1.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1732it [1:13:13,  1.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1747it [1:14:26,  1.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1762it [1:14:55,  1.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1786it [1:15:33,  1.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1801it [1:16:03,  1.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1816it [1:17:17,  1.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1831it [1:17:45,  1.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1846it [1:18:15,  1.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1861it [1:19:07,  1.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1876it [1:19:34,  1.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1891it [1:20:25,  1.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1906it [1:20:52,  2.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1921it [1:21:22,  1.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1936it [1:22:13,  1.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1951it [1:22:41,  1.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1966it [1:23:11,  2.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1981it [1:24:25,  1.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1996it [1:24:54,  1.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2011it [1:25:23,  1.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2026it [1:26:14,  1.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2041it [1:26:42,  1.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2065it [1:27:22,  2.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2080it [1:27:50,  2.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2095it [1:28:19,  1.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2110it [1:29:11,  1.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2129it [1:29:45,  1.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2144it [1:30:14,  1.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2159it [1:30:41,  2.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2174it [1:31:09,  1.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2195it [1:32:29,  1.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2210it [1:33:21,  1.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2225it [1:34:15,  1.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2240it [1:34:42,  2.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2255it [1:35:12,  2.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2270it [1:36:26,  1.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2285it [1:36:53,  2.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2300it [1:37:23,  1.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2315it [1:38:14,  1.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2330it [1:38:41,  2.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2345it [1:39:11,  1.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2360it [1:40:25,  1.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2375it [1:40:53,  1.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2390it [1:41:22,  1.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2405it [1:42:16,  1.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2420it [1:42:44,  1.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2435it [1:43:14,  1.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2450it [1:44:06,  1.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2465it [1:44:37,  1.22s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2480it [1:45:08,  1.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2495it [1:45:36,  2.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2510it [1:46:26,  1.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2525it [1:46:55,  1.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2540it [1:47:24,  1.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2555it [1:48:15,  1.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2570it [1:48:42,  2.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2585it [1:49:13,  1.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2600it [1:50:25,  1.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2615it [1:50:53,  1.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2630it [1:51:22,  2.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2645it [1:52:14,  1.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2660it [1:52:42,  1.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2675it [1:53:11,  1.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2690it [1:54:25,  1.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2705it [1:54:52,  2.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2720it [1:55:20,  2.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2735it [1:56:11,  1.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2750it [1:56:41,  1.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2765it [1:57:11,  1.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2780it [1:58:23,  1.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2795it [1:58:52,  1.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2810it [1:59:22,  1.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2825it [2:00:13,  1.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2840it [2:01:28,  1.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2855it [2:01:55,  1.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2870it [2:02:24,  2.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2885it [2:03:15,  1.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2900it [2:03:43,  1.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2915it [2:04:13,  1.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2930it [2:05:08,  1.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2945it [2:05:36,  1.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2960it [2:06:06,  2.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2975it [2:07:20,  1.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2990it [2:07:48,  1.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3005it [2:08:18,  1.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3020it [2:09:09,  1.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3035it [2:09:37,  1.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3050it [2:10:06,  1.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3065it [2:11:19,  1.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3080it [2:11:47,  1.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3095it [2:12:17,  1.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3112it [2:13:12,  1.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3127it [2:13:40,  1.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3142it [2:14:10,  1.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3157it [2:15:23,  1.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3172it [2:15:50,  2.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3191it [2:16:26,  1.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3206it [2:16:54,  1.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3221it [2:17:24,  1.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3236it [2:18:17,  1.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3251it [2:18:44,  1.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3266it [2:19:15,  1.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3281it [2:20:06,  1.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3296it [2:20:33,  2.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3311it [2:21:25,  1.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3326it [2:21:53,  1.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3341it [2:22:25,  1.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3356it [2:23:17,  1.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3393it [2:24:26,  2.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3408it [2:24:55,  1.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3423it [2:25:24,  1.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3438it [2:26:16,  1.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3453it [2:26:44,  1.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3468it [2:27:14,  1.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3483it [2:28:06,  1.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3498it [2:28:33,  2.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3513it [2:29:26,  1.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3528it [2:29:54,  1.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3543it [2:30:22,  2.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3558it [2:31:13,  1.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3573it [2:31:41,  1.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3588it [2:32:12,  1.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3603it [2:33:25,  1.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3618it [2:33:53,  1.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3637it [2:34:29,  1.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3652it [2:34:57,  1.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3667it [2:35:26,  1.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3682it [2:36:18,  1.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3697it [2:36:47,  1.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3712it [2:37:17,  1.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3727it [2:38:09,  1.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3742it [2:38:37,  1.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3762it [2:39:12,  1.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3777it [2:39:40,  2.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3792it [2:40:09,  1.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3807it [2:41:24,  1.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3822it [2:41:53,  1.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3837it [2:42:23,  1.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3852it [2:43:15,  1.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit exceeded. Retrying in 20 seconds...\n",
      "Rate limit exceeded. Retrying in 20 seconds...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3862it [2:44:03,  2.55s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "      index political_ideology_US\n",
      "0         0               neutral\n",
      "1         1               neutral\n",
      "2         2               liberal\n",
      "3         3               neutral\n",
      "4         4               neutral\n",
      "...     ...                   ...\n",
      "3857   3857               liberal\n",
      "3858   3858               neutral\n",
      "3859   3859               neutral\n",
      "3860   3860               neutral\n",
      "3861   3861               neutral\n",
      "\n",
      "[3862 rows x 2 columns]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "#check effect of adding a system prompt to GPT4-turbo:\n",
    "def load_json(path: str):\n",
    "    with open(path, encoding='utf-8') as fp:\n",
    "        return json.load(fp)\n",
    "    \n",
    "\n",
    "pubsphereazurerun6 = ['rationality_simple2']\n",
    "pubsphereazurerun7 = ['interactivity_acknowledgement_simple']\n",
    "pubsphereazurerun8 = ['incivility_simple2']\n",
    "pubsphereazurerun9 = ['political_ideology_US']\n",
    "\n",
    "\n",
    "chunked_result: typing.List[pd.DataFrame] = []\n",
    "\n",
    "for label, path in CFG.prompt_classify_files.items():\n",
    "    if label in pubsphereazurerun9: \n",
    "        template = load_json(path).get('template')\n",
    "        classes = load_json(path).get('classes')\n",
    "        for index, row in tqdm.tqdm(dataset[\"commentText\"].items()):\n",
    "            retry_count = 0\n",
    "            max_retries = 5\n",
    "            while retry_count < max_retries:\n",
    "                try: \n",
    "                    response = requests.post(\n",
    "                            url=api_endpoint,\n",
    "                            headers=headers,\n",
    "                            json={\n",
    "                                'model': MODELgpt4T,\n",
    "                                'messages': [\n",
    "                                    {\n",
    "                                        \"role\": \"system\",\n",
    "                                        \"content\": template\n",
    "                                    },\n",
    "                                    {\n",
    "                                        \"role\": \"user\",\n",
    "                                        \"content\": template.format(text=row)\n",
    "                                    }\n",
    "                                ],\n",
    "                                'temperature': temperature_0,  \n",
    "                                'seed': SEED,\n",
    "                                \"max_tokens\": MAX10\n",
    "                            }\n",
    "                        )  \n",
    "\n",
    "                    if response.status_code == 200:\n",
    "                        data_response = response.json()\n",
    "                        chunked_result.append(\n",
    "                        pd.DataFrame(\n",
    "                            data=[[index, classes.get(data_response[\"choices\"][0][\"message\"][\"content\"], None)]],                                \n",
    "                            columns=['index', label]\n",
    "                            )\n",
    "                        )\n",
    "                        break  # Exit the retry loop on success\n",
    "                    elif response.status_code == 429:\n",
    "                        retry_count += 1\n",
    "                        wait_time = 20\n",
    "                        print(f\"Rate limit exceeded. Retrying in {wait_time} seconds...\")\n",
    "                        time.sleep(wait_time)\n",
    "                    elif response.status_code == 500:\n",
    "                        retry_count += 1\n",
    "                        wait_time = 20\n",
    "                        print(f\"Failed to connect to API. Status code: {response.status_code}. Retrying in {wait_time} seconds...\")\n",
    "                        time.sleep(wait_time)\n",
    "                    else:\n",
    "                        print(f\"Failed to connect to API. Status code: {response.status_code}\")\n",
    "                        print(response.text)\n",
    "                        break\n",
    "                except requests.exceptions.RequestException as e:   \n",
    "                    print(f\"Failed to connect to API: {e}\")\n",
    "                    retry_count += 1\n",
    "                    wait_time = 60\n",
    "                    print(f\"Retrying in {wait_time} seconds...\")\n",
    "                    time.sleep(wait_time)                 \n",
    "                \n",
    "            #Add a pause of 1 minute after every 70 iterations\n",
    "            #if (index + 1) % 250 == 0:\n",
    "            #    print(\"Pausing for 10 seconds...\")\n",
    "            #    time.sleep(10)\n",
    "\n",
    "classifications9 = pd.concat(chunked_result, ignore_index=True)\n",
    "print(classifications9) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "d93ef444",
   "metadata": {},
   "outputs": [],
   "source": [
    "rationality_gpt4T_system = classifications4append.loc[:, [\"index\", \"rationality_simple2\"]].dropna()\n",
    "rationality_gpt4T_system.set_index('index', drop=True, inplace=True)\n",
    "dataset_w_pred_anon = dataset_w_pred_anon.join(rationality_gpt4T_system, rsuffix='_gpt4T_system')\n",
    "dataset_w_pred_anon.loc[:, 'rationality_simple2_gpt4T_system_dum'] = dataset_w_pred_anon.loc[:, 'rationality_simple2_gpt4T_system'].map({\"Yes\": 1, \"No\":0}).fillna(0).astype(int)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "ef33329a",
   "metadata": {},
   "outputs": [],
   "source": [
    "interactivity_gpt4T_system = classifications7.loc[:, [\"index\", \"interactivity_acknowledgement_simple\"]].dropna()\n",
    "interactivity_gpt4T_system.set_index('index', drop=True, inplace=True)\n",
    "dataset_w_pred_anon = dataset_w_pred_anon.join(interactivity_gpt4T_system, rsuffix='_gpt4T_system')\n",
    "dataset_w_pred_anon.loc[:, 'interactivity_acknowledgement_simple_gpt4T_system_dum'] = dataset_w_pred_anon.loc[:, 'interactivity_acknowledgement_simple_gpt4T_system'].map({\"Yes\": 1, \"No\":0}).fillna(0).astype(int)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "c1f1b9cf",
   "metadata": {},
   "outputs": [],
   "source": [
    "incivility_gpt4T_system = classifications8.loc[:, [\"index\", \"incivility_simple2\"]].dropna()\n",
    "incivility_gpt4T_system.set_index('index', drop=True, inplace=True)\n",
    "dataset_w_pred_anon = dataset_w_pred_anon.join(incivility_gpt4T_system, rsuffix='_gpt4T_system')\n",
    "dataset_w_pred_anon.loc[:, 'incivility_simple2_gpt4T_system_dum'] = dataset_w_pred_anon.loc[:, 'incivility_simple2_gpt4T_system'].map({\"Yes\": 1, \"No\":0}).fillna(0).astype(int)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "6f0c7e86",
   "metadata": {},
   "outputs": [],
   "source": [
    "political_ideology_US_gpt4T_system = classifications9.loc[:, [\"index\", \"political_ideology_US\"]].dropna()\n",
    "political_ideology_US_gpt4T_system.set_index('index', drop=True, inplace=True)\n",
    "dataset_w_pred_anon = dataset_w_pred_anon.join(political_ideology_US_gpt4T_system, rsuffix='_gpt4T_system')\n",
    "dataset_w_pred_anon.loc[:, 'political_conservative_US_gpt4T_system_dum'] = dataset_w_pred_anon.loc[:, 'political_ideology_US_gpt4T_system'].map({\"conservative\": 1}).fillna(0).astype(int)\n",
    "dataset_w_pred_anon.loc[:, 'political_liberal_US_gpt4T_system_dum'] = dataset_w_pred_anon.loc[:, 'political_ideology_US_gpt4T_system'].map({\"liberal\": 1}).fillna(0).astype(int)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "867a5a52",
   "metadata": {},
   "outputs": [],
   "source": [
    "rationality_gpt4T_system = classifications4.loc[:, [\"index\", \"rationality_simple2\"]].dropna()\n",
    "incivility_gpt4T_system = classifications4.loc[:, [\"index\", \"incivility_simple2\"]].dropna()\n",
    "interactivity_gpt4T_system = classifications4.loc[:, [\"index\", \"interactivity_acknowledgement_simple\"]].dropna()\n",
    "political_ideology_US_gpt4T_system = classifications4.loc[:, [\"index\", \"political_ideology_US\"]].dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "21fd8978",
   "metadata": {},
   "outputs": [],
   "source": [
    "rationality_gpt4T_system.set_index('index', drop=True, inplace=True)\n",
    "incivility_gpt4T_system.set_index('index', drop=True, inplace=True)\n",
    "interactivity_gpt4T_system.set_index('index', drop=True, inplace=True)\n",
    "political_ideology_US_gpt4T_system.set_index('index', drop=True, inplace=True)\n",
    "dataset_w_pred_anon = dataset_w_pred_anon.join(rationality_gpt4T_system, rsuffix='_gpt4T_system').join(incivility_gpt4T_system, rsuffix='_gpt4T_system').join(interactivity_gpt4T_system, rsuffix='_gpt4T_system').join(political_ideology_US_gpt4T_system, rsuffix='_gpt4T_system')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6def9bc6",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_w_pred_anon.loc[:, 'incivility_simple2_gpt4T_system_dum'] = dataset_w_pred_anon.loc[:, 'incivility_simple2_gpt4T_system'].map({\"Yes\": 1, \"No\":0}).fillna(0).astype(int)\n",
    "dataset_w_pred_anon.loc[:, 'rationality_simple2_gpt4T_system_dum'] = dataset_w_pred_anon.loc[:, 'rationality_simple2_gpt4T_system'].map({\"Yes\": 1, \"No\":0}).fillna(0).astype(int)\n",
    "dataset_w_pred_anon.loc[:, 'interactivity_acknowledgement_simple_gpt4T_system_dum'] = dataset_w_pred_anon.loc[:, 'interactivity_acknowledgement_simple_gpt4T_system'].map({\"Yes\": 1, \"No\":0}).fillna(0).astype(int)\n",
    "\n",
    "dataset_w_pred_anon.loc[:, 'political_conservative_US_gpt4T_system_dum'] = dataset_w_pred_anon.loc[:, 'political_ideology_US_gpt4T_system'].map({\"conservative\": 1}).fillna(0).astype(int)\n",
    "dataset_w_pred_anon.loc[:, 'political_liberal_US_gpt4T_system_dum'] = dataset_w_pred_anon.loc[:, 'political_ideology_US_gpt4T_system'].map({\"liberal\": 1}).fillna(0).astype(int)\n",
    "dataset_w_pred_anon.loc[:, 'political_opinion_US_gpt4T_system_dum'] = dataset_w_pred_anon.loc[:, 'political_ideology_US_gpt4T_system'].map({\"liberal\": 1, 'conservative': 1}).fillna(0).astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "d699ad48",
   "metadata": {},
   "outputs": [],
   "source": [
    "#save data:\n",
    "dataset_w_pred_anon.to_json('data/publicsphere/publicsphere.cardiff_prompt_classify_anon.json', orient=\"records\", force_ascii=False, indent=4)\n",
    "dataset_w_pred_anon.to_parquet('data/publicsphere/publicsphere.cardiff_prompt_classify_anon.parquet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "20fe9b95",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>StartDate</th>\n",
       "      <th>RecordedDate</th>\n",
       "      <th>IPAddress</th>\n",
       "      <th>Finished</th>\n",
       "      <th>Coder</th>\n",
       "      <th>ID</th>\n",
       "      <th>Mark_ID</th>\n",
       "      <th>Genre</th>\n",
       "      <th>topiccode</th>\n",
       "      <th>Platform</th>\n",
       "      <th>...</th>\n",
       "      <th>political_opinion_US_gpt4o_system_dum</th>\n",
       "      <th>rationality_simple2_gpt4T_system</th>\n",
       "      <th>rationality_simple2_gpt4T_system_dum</th>\n",
       "      <th>interactivity_acknowledgement_simple_gpt4T_system</th>\n",
       "      <th>interactivity_acknowledgement_simple_gpt4T_system_dum</th>\n",
       "      <th>incivility_simple2_gpt4T_system</th>\n",
       "      <th>incivility_simple2_gpt4T_system_dum</th>\n",
       "      <th>political_ideology_US_gpt4T_system</th>\n",
       "      <th>political_conservative_US_gpt4T_system_dum</th>\n",
       "      <th>political_liberal_US_gpt4T_system_dum</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5/30/2021 13:03:17</td>\n",
       "      <td>5/30/2021 13:04:17</td>\n",
       "      <td>62.194.51.29</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>UgyPHwv8G0cDE6-wEgl4AaABAg.8_0ZjJKSJty8_0kXGkAd2U</td>\n",
       "      <td>119</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>No</td>\n",
       "      <td>0</td>\n",
       "      <td>No</td>\n",
       "      <td>0</td>\n",
       "      <td>No</td>\n",
       "      <td>0</td>\n",
       "      <td>neutral</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>10/11/2021 10:34:05</td>\n",
       "      <td>10/11/2021 10:36:46</td>\n",
       "      <td>213.127.109.191</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>Ugx2WXq9UdV8mPPjejJ4AaABAg.8yHCKV0Boe58yYRxEQEF45</td>\n",
       "      <td>282</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>No</td>\n",
       "      <td>0</td>\n",
       "      <td>No</td>\n",
       "      <td>0</td>\n",
       "      <td>Yes</td>\n",
       "      <td>1</td>\n",
       "      <td>neutral</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>9/9/2021 18:49:48</td>\n",
       "      <td>9/9/2021 18:51:32</td>\n",
       "      <td>213.127.110.0</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>1110578710648890000</td>\n",
       "      <td>372</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>No</td>\n",
       "      <td>0</td>\n",
       "      <td>Yes</td>\n",
       "      <td>1</td>\n",
       "      <td>No</td>\n",
       "      <td>0</td>\n",
       "      <td>liberal</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6/6/2021 16:12:46</td>\n",
       "      <td>6/6/2021 16:16:16</td>\n",
       "      <td>213.127.76.145</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>UgwUPFScjJ0MCeaP2F54AaABAg.8lvp3fc9Euf8lvvgsUgEgV</td>\n",
       "      <td>769</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>No</td>\n",
       "      <td>0</td>\n",
       "      <td>No</td>\n",
       "      <td>0</td>\n",
       "      <td>No</td>\n",
       "      <td>0</td>\n",
       "      <td>neutral</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6/13/2021 13:25:49</td>\n",
       "      <td>6/13/2021 13:27:28</td>\n",
       "      <td>213.127.82.232</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>UgwWKCWtSJdFvjGHvTp4AaABAg.8kUC5dGrQ2H8kUDRihE2f3</td>\n",
       "      <td>1206</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>No</td>\n",
       "      <td>0</td>\n",
       "      <td>No</td>\n",
       "      <td>0</td>\n",
       "      <td>No</td>\n",
       "      <td>0</td>\n",
       "      <td>neutral</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3857</th>\n",
       "      <td>8/19/2021 14:50:13</td>\n",
       "      <td>8/19/2021 14:54:28</td>\n",
       "      <td>62.194.51.29</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>1152219467579100000</td>\n",
       "      <td>10000695</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>No</td>\n",
       "      <td>0</td>\n",
       "      <td>Yes</td>\n",
       "      <td>1</td>\n",
       "      <td>Yes</td>\n",
       "      <td>1</td>\n",
       "      <td>liberal</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3858</th>\n",
       "      <td>8/19/2021 15:10:27</td>\n",
       "      <td>8/19/2021 15:12:21</td>\n",
       "      <td>62.194.51.29</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>1085362296472430000</td>\n",
       "      <td>10007008</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>No</td>\n",
       "      <td>0</td>\n",
       "      <td>No</td>\n",
       "      <td>0</td>\n",
       "      <td>No</td>\n",
       "      <td>0</td>\n",
       "      <td>neutral</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3859</th>\n",
       "      <td>10/6/2021 16:08:39</td>\n",
       "      <td>10/6/2021 16:10:42</td>\n",
       "      <td>213.127.113.113</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>UghFY3QJ6nmT_ngCoAEC.7-H0Z7--wxd8goqpaPs-bl</td>\n",
       "      <td>20000102</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>No</td>\n",
       "      <td>0</td>\n",
       "      <td>No</td>\n",
       "      <td>0</td>\n",
       "      <td>No</td>\n",
       "      <td>0</td>\n",
       "      <td>neutral</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3860</th>\n",
       "      <td>10/15/2021 18:30:04</td>\n",
       "      <td>10/15/2021 18:35:40</td>\n",
       "      <td>213.127.109.191</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>UgyWabsmmnq3zam4DgZ4AaABAg</td>\n",
       "      <td>20000418</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>No</td>\n",
       "      <td>0</td>\n",
       "      <td>No</td>\n",
       "      <td>0</td>\n",
       "      <td>No</td>\n",
       "      <td>0</td>\n",
       "      <td>neutral</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3861</th>\n",
       "      <td>11/19/2021 17:49:17</td>\n",
       "      <td>11/19/2021 17:51:04</td>\n",
       "      <td>213.127.109.191</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>UgwPOHIDyICm10k0Mvx4AaABAg</td>\n",
       "      <td>20001003</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>Yes</td>\n",
       "      <td>1</td>\n",
       "      <td>No</td>\n",
       "      <td>0</td>\n",
       "      <td>No</td>\n",
       "      <td>0</td>\n",
       "      <td>neutral</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3862 rows × 248 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                StartDate         RecordedDate        IPAddress  Finished  \\\n",
       "0      5/30/2021 13:03:17   5/30/2021 13:04:17     62.194.51.29         1   \n",
       "1     10/11/2021 10:34:05  10/11/2021 10:36:46  213.127.109.191         1   \n",
       "2       9/9/2021 18:49:48    9/9/2021 18:51:32    213.127.110.0         1   \n",
       "3       6/6/2021 16:12:46    6/6/2021 16:16:16   213.127.76.145         1   \n",
       "4      6/13/2021 13:25:49   6/13/2021 13:27:28   213.127.82.232         1   \n",
       "...                   ...                  ...              ...       ...   \n",
       "3857   8/19/2021 14:50:13   8/19/2021 14:54:28     62.194.51.29         1   \n",
       "3858   8/19/2021 15:10:27   8/19/2021 15:12:21     62.194.51.29         1   \n",
       "3859   10/6/2021 16:08:39   10/6/2021 16:10:42  213.127.113.113         1   \n",
       "3860  10/15/2021 18:30:04  10/15/2021 18:35:40  213.127.109.191         1   \n",
       "3861  11/19/2021 17:49:17  11/19/2021 17:51:04  213.127.109.191         1   \n",
       "\n",
       "      Coder                                                 ID   Mark_ID  \\\n",
       "0         6  UgyPHwv8G0cDE6-wEgl4AaABAg.8_0ZjJKSJty8_0kXGkAd2U       119   \n",
       "1         6  Ugx2WXq9UdV8mPPjejJ4AaABAg.8yHCKV0Boe58yYRxEQEF45       282   \n",
       "2         6                                1110578710648890000       372   \n",
       "3         6  UgwUPFScjJ0MCeaP2F54AaABAg.8lvp3fc9Euf8lvvgsUgEgV       769   \n",
       "4         6  UgwWKCWtSJdFvjGHvTp4AaABAg.8kUC5dGrQ2H8kUDRihE2f3      1206   \n",
       "...     ...                                                ...       ...   \n",
       "3857      6                                1152219467579100000  10000695   \n",
       "3858      6                                1085362296472430000  10007008   \n",
       "3859      6        UghFY3QJ6nmT_ngCoAEC.7-H0Z7--wxd8goqpaPs-bl  20000102   \n",
       "3860      6                         UgyWabsmmnq3zam4DgZ4AaABAg  20000418   \n",
       "3861      6                         UgwPOHIDyICm10k0Mvx4AaABAg  20001003   \n",
       "\n",
       "      Genre  topiccode  Platform  ...  political_opinion_US_gpt4o_system_dum  \\\n",
       "0         0          0         1  ...                                      0   \n",
       "1         1          2         1  ...                                      0   \n",
       "2         2          4         2  ...                                      1   \n",
       "3         0          0         1  ...                                      0   \n",
       "4         0          0         1  ...                                      0   \n",
       "...     ...        ...       ...  ...                                    ...   \n",
       "3857      0          4         2  ...                                      0   \n",
       "3858      1          4         2  ...                                      0   \n",
       "3859      0          3         1  ...                                      0   \n",
       "3860      2          3         1  ...                                      0   \n",
       "3861      0          3         1  ...                                      0   \n",
       "\n",
       "     rationality_simple2_gpt4T_system  rationality_simple2_gpt4T_system_dum  \\\n",
       "0                                  No                                     0   \n",
       "1                                  No                                     0   \n",
       "2                                  No                                     0   \n",
       "3                                  No                                     0   \n",
       "4                                  No                                     0   \n",
       "...                               ...                                   ...   \n",
       "3857                               No                                     0   \n",
       "3858                               No                                     0   \n",
       "3859                               No                                     0   \n",
       "3860                               No                                     0   \n",
       "3861                              Yes                                     1   \n",
       "\n",
       "      interactivity_acknowledgement_simple_gpt4T_system  \\\n",
       "0                                                    No   \n",
       "1                                                    No   \n",
       "2                                                   Yes   \n",
       "3                                                    No   \n",
       "4                                                    No   \n",
       "...                                                 ...   \n",
       "3857                                                Yes   \n",
       "3858                                                 No   \n",
       "3859                                                 No   \n",
       "3860                                                 No   \n",
       "3861                                                 No   \n",
       "\n",
       "      interactivity_acknowledgement_simple_gpt4T_system_dum  \\\n",
       "0                                                     0       \n",
       "1                                                     0       \n",
       "2                                                     1       \n",
       "3                                                     0       \n",
       "4                                                     0       \n",
       "...                                                 ...       \n",
       "3857                                                  1       \n",
       "3858                                                  0       \n",
       "3859                                                  0       \n",
       "3860                                                  0       \n",
       "3861                                                  0       \n",
       "\n",
       "      incivility_simple2_gpt4T_system  incivility_simple2_gpt4T_system_dum  \\\n",
       "0                                  No                                    0   \n",
       "1                                 Yes                                    1   \n",
       "2                                  No                                    0   \n",
       "3                                  No                                    0   \n",
       "4                                  No                                    0   \n",
       "...                               ...                                  ...   \n",
       "3857                              Yes                                    1   \n",
       "3858                               No                                    0   \n",
       "3859                               No                                    0   \n",
       "3860                               No                                    0   \n",
       "3861                               No                                    0   \n",
       "\n",
       "      political_ideology_US_gpt4T_system  \\\n",
       "0                                neutral   \n",
       "1                                neutral   \n",
       "2                                liberal   \n",
       "3                                neutral   \n",
       "4                                neutral   \n",
       "...                                  ...   \n",
       "3857                             liberal   \n",
       "3858                             neutral   \n",
       "3859                             neutral   \n",
       "3860                             neutral   \n",
       "3861                             neutral   \n",
       "\n",
       "      political_conservative_US_gpt4T_system_dum  \\\n",
       "0                                              0   \n",
       "1                                              0   \n",
       "2                                              0   \n",
       "3                                              0   \n",
       "4                                              0   \n",
       "...                                          ...   \n",
       "3857                                           0   \n",
       "3858                                           0   \n",
       "3859                                           0   \n",
       "3860                                           0   \n",
       "3861                                           0   \n",
       "\n",
       "     political_liberal_US_gpt4T_system_dum  \n",
       "0                                        0  \n",
       "1                                        0  \n",
       "2                                        1  \n",
       "3                                        0  \n",
       "4                                        0  \n",
       "...                                    ...  \n",
       "3857                                     1  \n",
       "3858                                     0  \n",
       "3859                                     0  \n",
       "3860                                     0  \n",
       "3861                                     0  \n",
       "\n",
       "[3862 rows x 248 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset_w_pred_anon"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "7adf9481",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Inter-coder reliablity: classification_report(human as gold, llm as predicted), cohen_kappa(human, llm):\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.84      0.99      0.91      2541\n",
      "           1       0.84      0.14      0.24       548\n",
      "\n",
      "    accuracy                           0.84      3089\n",
      "   macro avg       0.84      0.57      0.57      3089\n",
      "weighted avg       0.84      0.84      0.79      3089\n",
      "\n",
      "cohen_kappa_score(RATIONALITY_DUMMY): 0.1973134272611098\n",
      "krippendorf(RATIONALITY_DUMMY): 0.15008726003490402\n",
      "percent_agreement(RATIONALITY_DUMMY): 84.23%\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.74      0.87      0.80      1567\n",
      "           1       0.83      0.69      0.75      1522\n",
      "\n",
      "    accuracy                           0.78      3089\n",
      "   macro avg       0.79      0.78      0.78      3089\n",
      "weighted avg       0.79      0.78      0.78      3089\n",
      "\n",
      "cohen_kappa_score(INCIVILITY_DUMMY): 0.5539779563940896\n",
      "krippendorf(INCIVILITY_DUMMY): 0.550684059222557\n",
      "percent_agreement(INCIVILITY_DUMMY): 77.76%\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.83      0.79      0.81      2297\n",
      "           1       0.46      0.53      0.49       792\n",
      "\n",
      "    accuracy                           0.72      3089\n",
      "   macro avg       0.65      0.66      0.65      3089\n",
      "weighted avg       0.74      0.72      0.73      3089\n",
      "\n",
      "cohen_kappa_score(INTERACTIVITY_DUMMY): 0.30274870092477246\n",
      "krippendorf(INTERACTIVITY_DUMMY): 0.30167055779909135\n",
      "percent_agreement(INTERACTIVITY_DUMMY): 72.16%\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.92      0.88      0.90      2440\n",
      "           1       0.61      0.71      0.65       649\n",
      "\n",
      "    accuracy                           0.84      3089\n",
      "   macro avg       0.76      0.79      0.78      3089\n",
      "weighted avg       0.85      0.84      0.85      3089\n",
      "\n",
      "cohen_kappa_score(LIBERAL_DUMMY): 0.5523167045587383\n",
      "krippendorf(LIBERAL_DUMMY): 0.5516661409759209\n",
      "percent_agreement(LIBERAL_DUMMY): 84.27%\n",
      "---\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.94      0.92      0.93      2611\n",
      "           1       0.60      0.66      0.63       478\n",
      "\n",
      "    accuracy                           0.88      3089\n",
      "   macro avg       0.77      0.79      0.78      3089\n",
      "weighted avg       0.88      0.88      0.88      3089\n",
      "\n",
      "cohen_kappa_score(CONSERVATIVE_DUMMY): 0.5587088334682128\n",
      "krippendorf(CONSERVATIVE_DUMMY): 0.5586152954808807\n",
      "percent_agreement(CONSERVATIVE_DUMMY): 88.02%\n"
     ]
    }
   ],
   "source": [
    "# check the effect on the training set:\n",
    "llm_human_column_pairs = [\n",
    "    (\"RATIONALITY_DUMMY\", \"rationality_simple2_gpt4T_system_dum\"),\n",
    "    (\"INCIVILITY_DUMMY\", \"incivility_simple2_gpt4T_system_dum\"),\n",
    "    (\"INTERACTIVITY_DUMMY\", \"interactivity_acknowledgement_simple_gpt4T_system_dum\"),\n",
    "    (\"LIBERAL_DUMMY\", \"political_liberal_US_gpt4T_system_dum\"),\n",
    "    (\"CONSERVATIVE_DUMMY\", \"political_conservative_US_gpt4T_system_dum\")  \n",
    "]\n",
    "\n",
    "print(\"Inter-coder reliablity: classification_report(human as gold, llm as predicted), cohen_kappa(human, llm):\")\n",
    "\n",
    "for human_col, llm_col in llm_human_column_pairs:\n",
    "    subset = dataset_w_pred_anon.loc[dataset_w_pred_anon.split == 'train', [human_col, llm_col]].dropna()\n",
    "    human = subset[human_col].tolist()\n",
    "    llm = subset[llm_col].tolist()\n",
    "\n",
    "    # Calculate percent agreement\n",
    "    agreement = np.sum(np.array(human) == np.array(llm))/len(np.array(human)) * 100\n",
    "\n",
    "    print(\"---\")\n",
    "    print(f\"{classification_report(human, llm)}\")\n",
    "    print(f\"cohen_kappa_score({human_col}): {cohen_kappa_score(human, llm)}\")\n",
    "    print(f\"krippendorf({human_col}): {krippendorff.alpha(np.array([human, llm]), level_of_measurement=\"nominal\")}\")\n",
    "    print(f\"percent_agreement({human_col}): {agreement:.2f}%\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "fe3a4761",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['StartDate',\n",
       " 'RecordedDate',\n",
       " 'IPAddress',\n",
       " 'Finished',\n",
       " 'Coder',\n",
       " 'ID',\n",
       " 'Mark_ID',\n",
       " 'Genre',\n",
       " 'topiccode',\n",
       " 'Platform',\n",
       " 'Anonymity',\n",
       " 'Anonymity_9_TEXT',\n",
       " 'codable',\n",
       " 'Interaction',\n",
       " 'Acknowledgement',\n",
       " 'TopicRelevance',\n",
       " 'Reasoning',\n",
       " 'BackgroundInfo',\n",
       " 'ExternalEvidence',\n",
       " 'ExternalEvidence_1_TEXT',\n",
       " 'Opinion',\n",
       " 'disagreement',\n",
       " 'Ideologicaldirection',\n",
       " 'Name_calling',\n",
       " 'Vulgarity',\n",
       " 'Attack_reputation',\n",
       " 'Question_Intelligenc',\n",
       " 'All_caps_function',\n",
       " 'Sarcasm_to_criticize',\n",
       " 'Individual_right',\n",
       " 'discrimination',\n",
       " 'Invoke_violence',\n",
       " 'Tone',\n",
       " 'INTERACTIVITY_DUMMY',\n",
       " 'RATIONALITY_DUMMY',\n",
       " 'HAS_OPINION_DUMMY',\n",
       " 'LIBERAL_NEUTRAL_CONSERVATIVE',\n",
       " 'LIBERAL_DUMMY',\n",
       " 'CONSERVATIVE_DUMMY',\n",
       " 'NAMECALLING_DUMMY',\n",
       " 'VULGAR_DUMMY',\n",
       " 'NAMECALLING_VULGAR_DUMMY',\n",
       " 'INCIVILITY_ORDINAL',\n",
       " 'INCIVILITY_DUMMY',\n",
       " 'INTOLERANCE_DUMMY',\n",
       " 'filter_$',\n",
       " 'IMPOLITENESS_DUMMY',\n",
       " 'showName',\n",
       " 'genre',\n",
       " 'Time_comment',\n",
       " 'likeCount_comment',\n",
       " 'entities',\n",
       " 'place',\n",
       " 'retweet_count',\n",
       " 'platform',\n",
       " 'retweeted',\n",
       " 'language',\n",
       " 'source',\n",
       " 'in_reply_to_status_id_str',\n",
       " 'in_reply_to_user_id_str',\n",
       " 'in_reply_to_screen_name',\n",
       " 'is_quote_status',\n",
       " 'videoTitle',\n",
       " 'description',\n",
       " 'Time_video',\n",
       " 'channelTitle',\n",
       " 'channelId',\n",
       " 'viewCount',\n",
       " 'dislikeCount_video',\n",
       " 'likeCount_video',\n",
       " 'date_difference',\n",
       " 'commentCount_video',\n",
       " 'replyCount_comment',\n",
       " 'topic',\n",
       " 'subscribers',\n",
       " 'HATELIST_FOCUSED_DUMMY',\n",
       " 'Time_comment_year',\n",
       " 'Time_video_year',\n",
       " 'interactivity_acknowledgement_old',\n",
       " 'rationality_external_evidence_old',\n",
       " 'rationality_topic_relevance_old',\n",
       " 'political_negativity_old',\n",
       " 'rationality_background_info_old',\n",
       " 'rationality_reasoning_old',\n",
       " 'topics',\n",
       " 'emotions',\n",
       " 'sentiment',\n",
       " 'irony',\n",
       " 'offensive',\n",
       " 'hate',\n",
       " 'political_ideology_US_old',\n",
       " 'offensive_dum',\n",
       " 'hate_dum',\n",
       " 'cardiff_incivil',\n",
       " 'rationality_reasoning_dum_old',\n",
       " 'rationality_background_info_dum_old',\n",
       " 'rationality_external_evidence_dum_old',\n",
       " 'rationality_prompt_dum_old',\n",
       " 'political_conservative_US_old',\n",
       " 'political_liberal_US_old',\n",
       " 'political_opinion_US_old',\n",
       " 'diversity_positiondum_old',\n",
       " 'diversity_disagreement',\n",
       " 'diversity_ideologicaldirection',\n",
       " 'diversity_position',\n",
       " 'incivility_allcaps',\n",
       " 'incivility_attackreputation',\n",
       " 'incivility_question_intelligence',\n",
       " 'incivility_sarcasm',\n",
       " 'incivility_namecalling_NA',\n",
       " 'incivility_vulgarity',\n",
       " 'diversity_positiondum_dum_old',\n",
       " 'diversity_disagreement_dum',\n",
       " 'diversity_position_dum',\n",
       " 'incivility_allcaps_dum',\n",
       " 'incivility_attackreputation_dum',\n",
       " 'incivility_question_intelligence_dum',\n",
       " 'incivility_sarcasm_dum',\n",
       " 'incivility_namecalling_dum_NA',\n",
       " 'incivility_vulgarity_dum',\n",
       " 'diversity_ideologicaldirection_dum',\n",
       " 'interactivity_acknowledgement_low',\n",
       " 'intolerance_discrimination',\n",
       " 'intolerance_rights',\n",
       " 'intolerance_violence',\n",
       " 'political_ideology_US_low',\n",
       " 'rationality_background_info_low',\n",
       " 'rationality_external_evidence_low',\n",
       " 'rationality_reasoning_low',\n",
       " 'rationality_topic_relevance_low',\n",
       " 'incivility_namecalling',\n",
       " 'rationality_background_info_updated',\n",
       " 'rationality_external_evidence_updated',\n",
       " 'interactivity_acknowledgement_dum',\n",
       " 'rationality_external_evidence_dum',\n",
       " 'rationality_topic_relevance_dum',\n",
       " 'rationality_background_info_low_dum',\n",
       " 'rationality_reasoning_dum',\n",
       " 'intolerance_discrimination_dum',\n",
       " 'intolerance_rights_dum',\n",
       " 'intolerance_violence_dum',\n",
       " 'incivility_namecalling_dum',\n",
       " 'political_conservative_US_dum',\n",
       " 'political_liberal_US_dum',\n",
       " 'political_opinion_US_dum',\n",
       " 'diversity_liberal_dum',\n",
       " 'rationality_background_info_updated_dum',\n",
       " 'rationality_external_evidence_updated_dum',\n",
       " 'rationality_prompt_dum',\n",
       " 'incivility_prompt_dum',\n",
       " 'hate_list_prompt_dum',\n",
       " 'rationality_background_info_run2',\n",
       " 'rationality_external_evidence_run2',\n",
       " 'rationality_external_evidence_edit',\n",
       " 'rationality_background_info_edit',\n",
       " 'interactivity_acknowledgement_updated',\n",
       " 'rationality_reasoning_edit',\n",
       " 'incivility_namecalling_updated',\n",
       " 'incivility_namecalling_edit',\n",
       " 'diversity_conservative_dum',\n",
       " 'interactivity_acknowledgement_updated_dum',\n",
       " 'incivility_namecalling_updated_dum',\n",
       " 'incivility_namecalling_edit_dum',\n",
       " 'rationality_reasoning_edit_dum',\n",
       " 'rationality_background_info_run2_dum',\n",
       " 'rationality_external_evidence_run2_dum',\n",
       " 'rationality_external_evidence_edit_dum',\n",
       " 'rationality_background_info_edit_dum',\n",
       " 'diversity_ideologicaldirection_long',\n",
       " 'diversity_conservative_long_dum',\n",
       " 'diversity_liberal_long_dum',\n",
       " 'diversity_disagreement_edit',\n",
       " 'diversity_disagreement_edit_dum',\n",
       " 'rationality_combine',\n",
       " 'rationality_combine_dum',\n",
       " 'incivility_combined',\n",
       " 'incivility_combine_dum',\n",
       " 'rationality_combine_exactexample',\n",
       " 'incivility_simple',\n",
       " 'rationality_simple',\n",
       " 'rationality_simple_dum',\n",
       " 'incivility_simple_dum',\n",
       " 'rationality_combine_exactexample_dum',\n",
       " 'incivility_simple2',\n",
       " 'interactivity_acknowledgement_simple',\n",
       " 'rationality_simple2',\n",
       " 'incivility_simple2_dum',\n",
       " 'rationality_simple2_dum',\n",
       " 'interactivity_acknowledgement_simple_dum',\n",
       " 'interactivity_acknowledgement_simple_small',\n",
       " 'rationality_simple2_small',\n",
       " 'interactivity_acknowledgement_simple_small2',\n",
       " 'incivility_simple2_small',\n",
       " 'political_ideology_US',\n",
       " 'incivility_simple2_small_dum',\n",
       " 'rationality_simple2_small_dum',\n",
       " 'interactivity_acknowledgement_simple_small_dum',\n",
       " 'interactivity_acknowledgement_simple_small2_dum',\n",
       " 'political_conservative_US_small_dum',\n",
       " 'political_liberal_US_small_dum',\n",
       " 'political_opinion_US_small_dum',\n",
       " 'interactivity_acknowledgement_simple_2',\n",
       " 'interactivity_acknowledgement_simple2_dum',\n",
       " 'interactivity_tot_simple2_dum',\n",
       " 'is_reply_sum',\n",
       " 'split',\n",
       " 'interactivity_acknowledgement_simple_gpt4o',\n",
       " 'incivility_simple2_gpt4o',\n",
       " 'rationality_simple2_gpt4o',\n",
       " 'political_ideology_US_gpt4o',\n",
       " 'incivility_simple2_gpt4o_dum',\n",
       " 'rationality_simple2_gpt4o_dum',\n",
       " 'interactivity_acknowledgement_simple_gpt4o_dum',\n",
       " 'political_conservative_US_gpt4o_dum',\n",
       " 'political_liberal_US_gpt4o_dum',\n",
       " 'political_opinion_US_gpt4o_dum',\n",
       " 'interactivity_acknowledgement_simple_gpt4o2',\n",
       " 'interactivity_acknowledgement_gpt4o_2_dum',\n",
       " 'interactivity_acknowledgement_simple_gpt4T',\n",
       " 'incivility_simple2_gpt4T',\n",
       " 'rationality_simple2_gpt4T',\n",
       " 'political_ideology_US_gpt4T',\n",
       " 'incivility_simple2_gpt4T_dum',\n",
       " 'rationality_simple2_gpt4T_dum',\n",
       " 'interactivity_acknowledgement_simple_gpt4T_dum',\n",
       " 'political_conservative_US_gpt4T_dum',\n",
       " 'political_liberal_US_gpt4T_dum',\n",
       " 'political_opinion_US_gpt4T_dum',\n",
       " 'interactivity_acknowledgement_simple_gpt4o_no_top_p',\n",
       " 'rationality_simple2_gpt4o_system',\n",
       " 'incivility_simple2_gpt4o_system',\n",
       " 'interactivity_acknowledgement_simple_gpt4o_system',\n",
       " 'political_ideology_US_gpt4o_system',\n",
       " 'incivility_simple2_gpt4o_system_dum',\n",
       " 'rationality_simple2_gpt4o_system_dum',\n",
       " 'interactivity_acknowledgement_simple_gpt4o_system_dum',\n",
       " 'political_conservative_US_gpt4o_system_dum',\n",
       " 'political_liberal_US_gpt4o_system_dum',\n",
       " 'political_opinion_US_gpt4o_system_dum',\n",
       " 'rationality_simple2_gpt4T_system',\n",
       " 'rationality_simple2_gpt4T_system_dum',\n",
       " 'interactivity_acknowledgement_simple_gpt4T_system',\n",
       " 'interactivity_acknowledgement_simple_gpt4T_system_dum',\n",
       " 'incivility_simple2_gpt4T_system',\n",
       " 'incivility_simple2_gpt4T_system_dum',\n",
       " 'political_ideology_US_gpt4T_system',\n",
       " 'political_conservative_US_gpt4T_system_dum',\n",
       " 'political_liberal_US_gpt4T_system_dum']"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#how do different GLLM annotations correlate:\n",
    "dataset_w_pred_anon.columns.to_list()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "95298bf3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['incivility_simple2_dum', 'incivility_simple2_small_dum', 'incivility_simple2_gpt4o_dum', 'incivility_simple2_gpt4T_dum', 'incivility_simple2_gpt4o_system_dum', 'incivility_simple2_gpt4T_system_dum']\n"
     ]
    }
   ],
   "source": [
    "#first for incivility:\n",
    "# Get all column names that contain 'incivility'\n",
    "incivility_columns = [col for col in dataset_w_pred_anon.columns if 'incivility_simple2' in col and col.endswith('_dum')]\n",
    "\n",
    "print(incivility_columns)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "647b6ce6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>incivility_simple2_dum</th>\n",
       "      <th>incivility_simple2_small_dum</th>\n",
       "      <th>incivility_simple2_gpt4o_dum</th>\n",
       "      <th>incivility_simple2_gpt4T_dum</th>\n",
       "      <th>incivility_simple2_gpt4o_system_dum</th>\n",
       "      <th>incivility_simple2_gpt4T_system_dum</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>incivility_simple2_dum</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.471924</td>\n",
       "      <td>0.559095</td>\n",
       "      <td>0.689808</td>\n",
       "      <td>0.528412</td>\n",
       "      <td>0.614689</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>incivility_simple2_small_dum</th>\n",
       "      <td>0.471924</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.680402</td>\n",
       "      <td>0.612758</td>\n",
       "      <td>0.699069</td>\n",
       "      <td>0.646075</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>incivility_simple2_gpt4o_dum</th>\n",
       "      <td>0.559095</td>\n",
       "      <td>0.680402</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.728996</td>\n",
       "      <td>0.879484</td>\n",
       "      <td>0.804484</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>incivility_simple2_gpt4T_dum</th>\n",
       "      <td>0.689808</td>\n",
       "      <td>0.612758</td>\n",
       "      <td>0.728996</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.700904</td>\n",
       "      <td>0.837150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>incivility_simple2_gpt4o_system_dum</th>\n",
       "      <td>0.528412</td>\n",
       "      <td>0.699069</td>\n",
       "      <td>0.879484</td>\n",
       "      <td>0.700904</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.792385</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>incivility_simple2_gpt4T_system_dum</th>\n",
       "      <td>0.614689</td>\n",
       "      <td>0.646075</td>\n",
       "      <td>0.804484</td>\n",
       "      <td>0.837150</td>\n",
       "      <td>0.792385</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     incivility_simple2_dum  \\\n",
       "incivility_simple2_dum                             1.000000   \n",
       "incivility_simple2_small_dum                       0.471924   \n",
       "incivility_simple2_gpt4o_dum                       0.559095   \n",
       "incivility_simple2_gpt4T_dum                       0.689808   \n",
       "incivility_simple2_gpt4o_system_dum                0.528412   \n",
       "incivility_simple2_gpt4T_system_dum                0.614689   \n",
       "\n",
       "                                     incivility_simple2_small_dum  \\\n",
       "incivility_simple2_dum                                   0.471924   \n",
       "incivility_simple2_small_dum                             1.000000   \n",
       "incivility_simple2_gpt4o_dum                             0.680402   \n",
       "incivility_simple2_gpt4T_dum                             0.612758   \n",
       "incivility_simple2_gpt4o_system_dum                      0.699069   \n",
       "incivility_simple2_gpt4T_system_dum                      0.646075   \n",
       "\n",
       "                                     incivility_simple2_gpt4o_dum  \\\n",
       "incivility_simple2_dum                                   0.559095   \n",
       "incivility_simple2_small_dum                             0.680402   \n",
       "incivility_simple2_gpt4o_dum                             1.000000   \n",
       "incivility_simple2_gpt4T_dum                             0.728996   \n",
       "incivility_simple2_gpt4o_system_dum                      0.879484   \n",
       "incivility_simple2_gpt4T_system_dum                      0.804484   \n",
       "\n",
       "                                     incivility_simple2_gpt4T_dum  \\\n",
       "incivility_simple2_dum                                   0.689808   \n",
       "incivility_simple2_small_dum                             0.612758   \n",
       "incivility_simple2_gpt4o_dum                             0.728996   \n",
       "incivility_simple2_gpt4T_dum                             1.000000   \n",
       "incivility_simple2_gpt4o_system_dum                      0.700904   \n",
       "incivility_simple2_gpt4T_system_dum                      0.837150   \n",
       "\n",
       "                                     incivility_simple2_gpt4o_system_dum  \\\n",
       "incivility_simple2_dum                                          0.528412   \n",
       "incivility_simple2_small_dum                                    0.699069   \n",
       "incivility_simple2_gpt4o_dum                                    0.879484   \n",
       "incivility_simple2_gpt4T_dum                                    0.700904   \n",
       "incivility_simple2_gpt4o_system_dum                             1.000000   \n",
       "incivility_simple2_gpt4T_system_dum                             0.792385   \n",
       "\n",
       "                                     incivility_simple2_gpt4T_system_dum  \n",
       "incivility_simple2_dum                                          0.614689  \n",
       "incivility_simple2_small_dum                                    0.646075  \n",
       "incivility_simple2_gpt4o_dum                                    0.804484  \n",
       "incivility_simple2_gpt4T_dum                                    0.837150  \n",
       "incivility_simple2_gpt4o_system_dum                             0.792385  \n",
       "incivility_simple2_gpt4T_system_dum                             1.000000  "
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#check correlation between these GLLM annotations columns:\n",
    "incivility_subset = dataset_w_pred_anon.loc[:, incivility_columns].dropna()\n",
    "incivility_subset.corr()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "fe861de0",
   "metadata": {},
   "outputs": [],
   "source": [
    "#reasonable high correlation between Llama3.1_70b, 8b, GPT4o, GPT4T, GPT4o_system and GPT4T_system\n",
    "#correlation higher among GPT models than between GPT and Llama, lowest correlation between Llama70b and Llama8b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "e0a4da23",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['interactivity_acknowledgement_simple_dum', 'interactivity_acknowledgement_simple_small_dum', 'interactivity_acknowledgement_simple_small2_dum', 'interactivity_acknowledgement_simple2_dum', 'interactivity_acknowledgement_simple_gpt4o_dum', 'interactivity_acknowledgement_simple_gpt4T_dum', 'interactivity_acknowledgement_simple_gpt4o_system_dum', 'interactivity_acknowledgement_simple_gpt4T_system_dum']\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>interactivity_acknowledgement_simple_dum</th>\n",
       "      <th>interactivity_acknowledgement_simple_small_dum</th>\n",
       "      <th>interactivity_acknowledgement_simple_small2_dum</th>\n",
       "      <th>interactivity_acknowledgement_simple2_dum</th>\n",
       "      <th>interactivity_acknowledgement_simple_gpt4o_dum</th>\n",
       "      <th>interactivity_acknowledgement_simple_gpt4T_dum</th>\n",
       "      <th>interactivity_acknowledgement_simple_gpt4o_system_dum</th>\n",
       "      <th>interactivity_acknowledgement_simple_gpt4T_system_dum</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>interactivity_acknowledgement_simple_dum</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.475591</td>\n",
       "      <td>0.460837</td>\n",
       "      <td>0.793577</td>\n",
       "      <td>0.501588</td>\n",
       "      <td>0.528823</td>\n",
       "      <td>0.506538</td>\n",
       "      <td>0.558459</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>interactivity_acknowledgement_simple_small_dum</th>\n",
       "      <td>0.475591</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.668649</td>\n",
       "      <td>0.436232</td>\n",
       "      <td>0.324138</td>\n",
       "      <td>0.354690</td>\n",
       "      <td>0.340700</td>\n",
       "      <td>0.371424</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>interactivity_acknowledgement_simple_small2_dum</th>\n",
       "      <td>0.460837</td>\n",
       "      <td>0.668649</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.508801</td>\n",
       "      <td>0.433949</td>\n",
       "      <td>0.482587</td>\n",
       "      <td>0.469248</td>\n",
       "      <td>0.478510</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>interactivity_acknowledgement_simple2_dum</th>\n",
       "      <td>0.793577</td>\n",
       "      <td>0.436232</td>\n",
       "      <td>0.508801</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.535562</td>\n",
       "      <td>0.594761</td>\n",
       "      <td>0.560263</td>\n",
       "      <td>0.586422</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>interactivity_acknowledgement_simple_gpt4o_dum</th>\n",
       "      <td>0.501588</td>\n",
       "      <td>0.324138</td>\n",
       "      <td>0.433949</td>\n",
       "      <td>0.535562</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.755869</td>\n",
       "      <td>0.794506</td>\n",
       "      <td>0.716759</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>interactivity_acknowledgement_simple_gpt4T_dum</th>\n",
       "      <td>0.528823</td>\n",
       "      <td>0.354690</td>\n",
       "      <td>0.482587</td>\n",
       "      <td>0.594761</td>\n",
       "      <td>0.755869</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.804409</td>\n",
       "      <td>0.795535</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>interactivity_acknowledgement_simple_gpt4o_system_dum</th>\n",
       "      <td>0.506538</td>\n",
       "      <td>0.340700</td>\n",
       "      <td>0.469248</td>\n",
       "      <td>0.560263</td>\n",
       "      <td>0.794506</td>\n",
       "      <td>0.804409</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.746010</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>interactivity_acknowledgement_simple_gpt4T_system_dum</th>\n",
       "      <td>0.558459</td>\n",
       "      <td>0.371424</td>\n",
       "      <td>0.478510</td>\n",
       "      <td>0.586422</td>\n",
       "      <td>0.716759</td>\n",
       "      <td>0.795535</td>\n",
       "      <td>0.746010</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    interactivity_acknowledgement_simple_dum  \\\n",
       "interactivity_acknowledgement_simple_dum                                            1.000000   \n",
       "interactivity_acknowledgement_simple_small_dum                                      0.475591   \n",
       "interactivity_acknowledgement_simple_small2_dum                                     0.460837   \n",
       "interactivity_acknowledgement_simple2_dum                                           0.793577   \n",
       "interactivity_acknowledgement_simple_gpt4o_dum                                      0.501588   \n",
       "interactivity_acknowledgement_simple_gpt4T_dum                                      0.528823   \n",
       "interactivity_acknowledgement_simple_gpt4o_syst...                                  0.506538   \n",
       "interactivity_acknowledgement_simple_gpt4T_syst...                                  0.558459   \n",
       "\n",
       "                                                    interactivity_acknowledgement_simple_small_dum  \\\n",
       "interactivity_acknowledgement_simple_dum                                                  0.475591   \n",
       "interactivity_acknowledgement_simple_small_dum                                            1.000000   \n",
       "interactivity_acknowledgement_simple_small2_dum                                           0.668649   \n",
       "interactivity_acknowledgement_simple2_dum                                                 0.436232   \n",
       "interactivity_acknowledgement_simple_gpt4o_dum                                            0.324138   \n",
       "interactivity_acknowledgement_simple_gpt4T_dum                                            0.354690   \n",
       "interactivity_acknowledgement_simple_gpt4o_syst...                                        0.340700   \n",
       "interactivity_acknowledgement_simple_gpt4T_syst...                                        0.371424   \n",
       "\n",
       "                                                    interactivity_acknowledgement_simple_small2_dum  \\\n",
       "interactivity_acknowledgement_simple_dum                                                   0.460837   \n",
       "interactivity_acknowledgement_simple_small_dum                                             0.668649   \n",
       "interactivity_acknowledgement_simple_small2_dum                                            1.000000   \n",
       "interactivity_acknowledgement_simple2_dum                                                  0.508801   \n",
       "interactivity_acknowledgement_simple_gpt4o_dum                                             0.433949   \n",
       "interactivity_acknowledgement_simple_gpt4T_dum                                             0.482587   \n",
       "interactivity_acknowledgement_simple_gpt4o_syst...                                         0.469248   \n",
       "interactivity_acknowledgement_simple_gpt4T_syst...                                         0.478510   \n",
       "\n",
       "                                                    interactivity_acknowledgement_simple2_dum  \\\n",
       "interactivity_acknowledgement_simple_dum                                             0.793577   \n",
       "interactivity_acknowledgement_simple_small_dum                                       0.436232   \n",
       "interactivity_acknowledgement_simple_small2_dum                                      0.508801   \n",
       "interactivity_acknowledgement_simple2_dum                                            1.000000   \n",
       "interactivity_acknowledgement_simple_gpt4o_dum                                       0.535562   \n",
       "interactivity_acknowledgement_simple_gpt4T_dum                                       0.594761   \n",
       "interactivity_acknowledgement_simple_gpt4o_syst...                                   0.560263   \n",
       "interactivity_acknowledgement_simple_gpt4T_syst...                                   0.586422   \n",
       "\n",
       "                                                    interactivity_acknowledgement_simple_gpt4o_dum  \\\n",
       "interactivity_acknowledgement_simple_dum                                                  0.501588   \n",
       "interactivity_acknowledgement_simple_small_dum                                            0.324138   \n",
       "interactivity_acknowledgement_simple_small2_dum                                           0.433949   \n",
       "interactivity_acknowledgement_simple2_dum                                                 0.535562   \n",
       "interactivity_acknowledgement_simple_gpt4o_dum                                            1.000000   \n",
       "interactivity_acknowledgement_simple_gpt4T_dum                                            0.755869   \n",
       "interactivity_acknowledgement_simple_gpt4o_syst...                                        0.794506   \n",
       "interactivity_acknowledgement_simple_gpt4T_syst...                                        0.716759   \n",
       "\n",
       "                                                    interactivity_acknowledgement_simple_gpt4T_dum  \\\n",
       "interactivity_acknowledgement_simple_dum                                                  0.528823   \n",
       "interactivity_acknowledgement_simple_small_dum                                            0.354690   \n",
       "interactivity_acknowledgement_simple_small2_dum                                           0.482587   \n",
       "interactivity_acknowledgement_simple2_dum                                                 0.594761   \n",
       "interactivity_acknowledgement_simple_gpt4o_dum                                            0.755869   \n",
       "interactivity_acknowledgement_simple_gpt4T_dum                                            1.000000   \n",
       "interactivity_acknowledgement_simple_gpt4o_syst...                                        0.804409   \n",
       "interactivity_acknowledgement_simple_gpt4T_syst...                                        0.795535   \n",
       "\n",
       "                                                    interactivity_acknowledgement_simple_gpt4o_system_dum  \\\n",
       "interactivity_acknowledgement_simple_dum                                                     0.506538       \n",
       "interactivity_acknowledgement_simple_small_dum                                               0.340700       \n",
       "interactivity_acknowledgement_simple_small2_dum                                              0.469248       \n",
       "interactivity_acknowledgement_simple2_dum                                                    0.560263       \n",
       "interactivity_acknowledgement_simple_gpt4o_dum                                               0.794506       \n",
       "interactivity_acknowledgement_simple_gpt4T_dum                                               0.804409       \n",
       "interactivity_acknowledgement_simple_gpt4o_syst...                                           1.000000       \n",
       "interactivity_acknowledgement_simple_gpt4T_syst...                                           0.746010       \n",
       "\n",
       "                                                    interactivity_acknowledgement_simple_gpt4T_system_dum  \n",
       "interactivity_acknowledgement_simple_dum                                                     0.558459      \n",
       "interactivity_acknowledgement_simple_small_dum                                               0.371424      \n",
       "interactivity_acknowledgement_simple_small2_dum                                              0.478510      \n",
       "interactivity_acknowledgement_simple2_dum                                                    0.586422      \n",
       "interactivity_acknowledgement_simple_gpt4o_dum                                               0.716759      \n",
       "interactivity_acknowledgement_simple_gpt4T_dum                                               0.795535      \n",
       "interactivity_acknowledgement_simple_gpt4o_syst...                                           0.746010      \n",
       "interactivity_acknowledgement_simple_gpt4T_syst...                                           1.000000      "
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#and for interactivity_acknowledgement:\n",
    "# Get all column names that contain 'interactivity_acknowledgement'\n",
    "interactivity_columns = [col for col in dataset_w_pred_anon.columns if 'interactivity_acknowledgement_simple' in col and col.endswith('_dum')]\n",
    "#check correlation between these GLLM annotations columns:\n",
    "interactivity_subset = dataset_w_pred_anon.loc[:, interactivity_columns].dropna()\n",
    "interactivity_subset.corr()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0ef645a3",
   "metadata": {},
   "outputs": [],
   "source": [
    "#much lower correlation between the different GLLM annotations for interactivity_acknowledgement than for incivility\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "f7262b61",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>rationality_simple2_dum</th>\n",
       "      <th>rationality_simple2_small_dum</th>\n",
       "      <th>rationality_simple2_gpt4o_dum</th>\n",
       "      <th>rationality_simple2_gpt4T_dum</th>\n",
       "      <th>rationality_simple2_gpt4o_system_dum</th>\n",
       "      <th>rationality_simple2_gpt4T_system_dum</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>rationality_simple2_dum</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.540533</td>\n",
       "      <td>0.449398</td>\n",
       "      <td>0.505340</td>\n",
       "      <td>0.589630</td>\n",
       "      <td>0.532775</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rationality_simple2_small_dum</th>\n",
       "      <td>0.540533</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.543429</td>\n",
       "      <td>0.586489</td>\n",
       "      <td>0.575136</td>\n",
       "      <td>0.592249</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rationality_simple2_gpt4o_dum</th>\n",
       "      <td>0.449398</td>\n",
       "      <td>0.543429</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.665507</td>\n",
       "      <td>0.719450</td>\n",
       "      <td>0.678654</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rationality_simple2_gpt4T_dum</th>\n",
       "      <td>0.505340</td>\n",
       "      <td>0.586489</td>\n",
       "      <td>0.665507</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.696528</td>\n",
       "      <td>0.796434</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rationality_simple2_gpt4o_system_dum</th>\n",
       "      <td>0.589630</td>\n",
       "      <td>0.575136</td>\n",
       "      <td>0.719450</td>\n",
       "      <td>0.696528</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.720443</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rationality_simple2_gpt4T_system_dum</th>\n",
       "      <td>0.532775</td>\n",
       "      <td>0.592249</td>\n",
       "      <td>0.678654</td>\n",
       "      <td>0.796434</td>\n",
       "      <td>0.720443</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      rationality_simple2_dum  \\\n",
       "rationality_simple2_dum                              1.000000   \n",
       "rationality_simple2_small_dum                        0.540533   \n",
       "rationality_simple2_gpt4o_dum                        0.449398   \n",
       "rationality_simple2_gpt4T_dum                        0.505340   \n",
       "rationality_simple2_gpt4o_system_dum                 0.589630   \n",
       "rationality_simple2_gpt4T_system_dum                 0.532775   \n",
       "\n",
       "                                      rationality_simple2_small_dum  \\\n",
       "rationality_simple2_dum                                    0.540533   \n",
       "rationality_simple2_small_dum                              1.000000   \n",
       "rationality_simple2_gpt4o_dum                              0.543429   \n",
       "rationality_simple2_gpt4T_dum                              0.586489   \n",
       "rationality_simple2_gpt4o_system_dum                       0.575136   \n",
       "rationality_simple2_gpt4T_system_dum                       0.592249   \n",
       "\n",
       "                                      rationality_simple2_gpt4o_dum  \\\n",
       "rationality_simple2_dum                                    0.449398   \n",
       "rationality_simple2_small_dum                              0.543429   \n",
       "rationality_simple2_gpt4o_dum                              1.000000   \n",
       "rationality_simple2_gpt4T_dum                              0.665507   \n",
       "rationality_simple2_gpt4o_system_dum                       0.719450   \n",
       "rationality_simple2_gpt4T_system_dum                       0.678654   \n",
       "\n",
       "                                      rationality_simple2_gpt4T_dum  \\\n",
       "rationality_simple2_dum                                    0.505340   \n",
       "rationality_simple2_small_dum                              0.586489   \n",
       "rationality_simple2_gpt4o_dum                              0.665507   \n",
       "rationality_simple2_gpt4T_dum                              1.000000   \n",
       "rationality_simple2_gpt4o_system_dum                       0.696528   \n",
       "rationality_simple2_gpt4T_system_dum                       0.796434   \n",
       "\n",
       "                                      rationality_simple2_gpt4o_system_dum  \\\n",
       "rationality_simple2_dum                                           0.589630   \n",
       "rationality_simple2_small_dum                                     0.575136   \n",
       "rationality_simple2_gpt4o_dum                                     0.719450   \n",
       "rationality_simple2_gpt4T_dum                                     0.696528   \n",
       "rationality_simple2_gpt4o_system_dum                              1.000000   \n",
       "rationality_simple2_gpt4T_system_dum                              0.720443   \n",
       "\n",
       "                                      rationality_simple2_gpt4T_system_dum  \n",
       "rationality_simple2_dum                                           0.532775  \n",
       "rationality_simple2_small_dum                                     0.592249  \n",
       "rationality_simple2_gpt4o_dum                                     0.678654  \n",
       "rationality_simple2_gpt4T_dum                                     0.796434  \n",
       "rationality_simple2_gpt4o_system_dum                              0.720443  \n",
       "rationality_simple2_gpt4T_system_dum                              1.000000  "
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#for rationality:\n",
    "# Get all column names that contain 'rationality'\n",
    "rationality_columns = [col for col in dataset_w_pred_anon.columns if 'rationality_simple2' in col and col.endswith('_dum')]\n",
    "#check correlation between these GLLM annotations columns:\n",
    "rationality_subset = dataset_w_pred_anon.loc[:, rationality_columns].dropna()\n",
    "rationality_subset.corr()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e1bcc8c4",
   "metadata": {},
   "outputs": [],
   "source": [
    "#correlations among rationality similar to interactivity_acknowledgement, lower than for incivility"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "1b10abb8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>political_liberal_US_dum</th>\n",
       "      <th>political_liberal_US_small_dum</th>\n",
       "      <th>political_liberal_US_gpt4o_dum</th>\n",
       "      <th>political_liberal_US_gpt4T_dum</th>\n",
       "      <th>political_liberal_US_gpt4o_system_dum</th>\n",
       "      <th>political_liberal_US_gpt4T_system_dum</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>political_liberal_US_dum</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.614080</td>\n",
       "      <td>0.706788</td>\n",
       "      <td>0.673822</td>\n",
       "      <td>0.723958</td>\n",
       "      <td>0.740332</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>political_liberal_US_small_dum</th>\n",
       "      <td>0.614080</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.634211</td>\n",
       "      <td>0.541435</td>\n",
       "      <td>0.625072</td>\n",
       "      <td>0.597222</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>political_liberal_US_gpt4o_dum</th>\n",
       "      <td>0.706788</td>\n",
       "      <td>0.634211</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.658144</td>\n",
       "      <td>0.884172</td>\n",
       "      <td>0.761703</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>political_liberal_US_gpt4T_dum</th>\n",
       "      <td>0.673822</td>\n",
       "      <td>0.541435</td>\n",
       "      <td>0.658144</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.679160</td>\n",
       "      <td>0.822536</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>political_liberal_US_gpt4o_system_dum</th>\n",
       "      <td>0.723958</td>\n",
       "      <td>0.625072</td>\n",
       "      <td>0.884172</td>\n",
       "      <td>0.679160</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.791126</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>political_liberal_US_gpt4T_system_dum</th>\n",
       "      <td>0.740332</td>\n",
       "      <td>0.597222</td>\n",
       "      <td>0.761703</td>\n",
       "      <td>0.822536</td>\n",
       "      <td>0.791126</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       political_liberal_US_dum  \\\n",
       "political_liberal_US_dum                               1.000000   \n",
       "political_liberal_US_small_dum                         0.614080   \n",
       "political_liberal_US_gpt4o_dum                         0.706788   \n",
       "political_liberal_US_gpt4T_dum                         0.673822   \n",
       "political_liberal_US_gpt4o_system_dum                  0.723958   \n",
       "political_liberal_US_gpt4T_system_dum                  0.740332   \n",
       "\n",
       "                                       political_liberal_US_small_dum  \\\n",
       "political_liberal_US_dum                                     0.614080   \n",
       "political_liberal_US_small_dum                               1.000000   \n",
       "political_liberal_US_gpt4o_dum                               0.634211   \n",
       "political_liberal_US_gpt4T_dum                               0.541435   \n",
       "political_liberal_US_gpt4o_system_dum                        0.625072   \n",
       "political_liberal_US_gpt4T_system_dum                        0.597222   \n",
       "\n",
       "                                       political_liberal_US_gpt4o_dum  \\\n",
       "political_liberal_US_dum                                     0.706788   \n",
       "political_liberal_US_small_dum                               0.634211   \n",
       "political_liberal_US_gpt4o_dum                               1.000000   \n",
       "political_liberal_US_gpt4T_dum                               0.658144   \n",
       "political_liberal_US_gpt4o_system_dum                        0.884172   \n",
       "political_liberal_US_gpt4T_system_dum                        0.761703   \n",
       "\n",
       "                                       political_liberal_US_gpt4T_dum  \\\n",
       "political_liberal_US_dum                                     0.673822   \n",
       "political_liberal_US_small_dum                               0.541435   \n",
       "political_liberal_US_gpt4o_dum                               0.658144   \n",
       "political_liberal_US_gpt4T_dum                               1.000000   \n",
       "political_liberal_US_gpt4o_system_dum                        0.679160   \n",
       "political_liberal_US_gpt4T_system_dum                        0.822536   \n",
       "\n",
       "                                       political_liberal_US_gpt4o_system_dum  \\\n",
       "political_liberal_US_dum                                            0.723958   \n",
       "political_liberal_US_small_dum                                      0.625072   \n",
       "political_liberal_US_gpt4o_dum                                      0.884172   \n",
       "political_liberal_US_gpt4T_dum                                      0.679160   \n",
       "political_liberal_US_gpt4o_system_dum                               1.000000   \n",
       "political_liberal_US_gpt4T_system_dum                               0.791126   \n",
       "\n",
       "                                       political_liberal_US_gpt4T_system_dum  \n",
       "political_liberal_US_dum                                            0.740332  \n",
       "political_liberal_US_small_dum                                      0.597222  \n",
       "political_liberal_US_gpt4o_dum                                      0.761703  \n",
       "political_liberal_US_gpt4T_dum                                      0.822536  \n",
       "political_liberal_US_gpt4o_system_dum                               0.791126  \n",
       "political_liberal_US_gpt4T_system_dum                               1.000000  "
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#for diversity liberal:\n",
    "# Get all column names that contain 'liberal'\n",
    "liberal_columns = [col for col in dataset_w_pred_anon.columns if 'political_liberal_US' in col and col.endswith('_dum')]\n",
    "#check correlation between these GLLM annotations:\n",
    "liberal_subset = dataset_w_pred_anon.loc[:, liberal_columns].dropna()\n",
    "liberal_subset.corr()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "3a79cca7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>political_conservative_US_dum</th>\n",
       "      <th>political_conservative_US_small_dum</th>\n",
       "      <th>political_conservative_US_gpt4o_dum</th>\n",
       "      <th>political_conservative_US_gpt4T_dum</th>\n",
       "      <th>political_conservative_US_gpt4o_system_dum</th>\n",
       "      <th>political_conservative_US_gpt4T_system_dum</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>political_conservative_US_dum</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.512085</td>\n",
       "      <td>0.772637</td>\n",
       "      <td>0.648838</td>\n",
       "      <td>0.756985</td>\n",
       "      <td>0.777826</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>political_conservative_US_small_dum</th>\n",
       "      <td>0.512085</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.460367</td>\n",
       "      <td>0.399020</td>\n",
       "      <td>0.446311</td>\n",
       "      <td>0.444957</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>political_conservative_US_gpt4o_dum</th>\n",
       "      <td>0.772637</td>\n",
       "      <td>0.460367</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.689458</td>\n",
       "      <td>0.877906</td>\n",
       "      <td>0.799466</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>political_conservative_US_gpt4T_dum</th>\n",
       "      <td>0.648838</td>\n",
       "      <td>0.399020</td>\n",
       "      <td>0.689458</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.685075</td>\n",
       "      <td>0.788238</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>political_conservative_US_gpt4o_system_dum</th>\n",
       "      <td>0.756985</td>\n",
       "      <td>0.446311</td>\n",
       "      <td>0.877906</td>\n",
       "      <td>0.685075</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.791994</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>political_conservative_US_gpt4T_system_dum</th>\n",
       "      <td>0.777826</td>\n",
       "      <td>0.444957</td>\n",
       "      <td>0.799466</td>\n",
       "      <td>0.788238</td>\n",
       "      <td>0.791994</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            political_conservative_US_dum  \\\n",
       "political_conservative_US_dum                                    1.000000   \n",
       "political_conservative_US_small_dum                              0.512085   \n",
       "political_conservative_US_gpt4o_dum                              0.772637   \n",
       "political_conservative_US_gpt4T_dum                              0.648838   \n",
       "political_conservative_US_gpt4o_system_dum                       0.756985   \n",
       "political_conservative_US_gpt4T_system_dum                       0.777826   \n",
       "\n",
       "                                            political_conservative_US_small_dum  \\\n",
       "political_conservative_US_dum                                          0.512085   \n",
       "political_conservative_US_small_dum                                    1.000000   \n",
       "political_conservative_US_gpt4o_dum                                    0.460367   \n",
       "political_conservative_US_gpt4T_dum                                    0.399020   \n",
       "political_conservative_US_gpt4o_system_dum                             0.446311   \n",
       "political_conservative_US_gpt4T_system_dum                             0.444957   \n",
       "\n",
       "                                            political_conservative_US_gpt4o_dum  \\\n",
       "political_conservative_US_dum                                          0.772637   \n",
       "political_conservative_US_small_dum                                    0.460367   \n",
       "political_conservative_US_gpt4o_dum                                    1.000000   \n",
       "political_conservative_US_gpt4T_dum                                    0.689458   \n",
       "political_conservative_US_gpt4o_system_dum                             0.877906   \n",
       "political_conservative_US_gpt4T_system_dum                             0.799466   \n",
       "\n",
       "                                            political_conservative_US_gpt4T_dum  \\\n",
       "political_conservative_US_dum                                          0.648838   \n",
       "political_conservative_US_small_dum                                    0.399020   \n",
       "political_conservative_US_gpt4o_dum                                    0.689458   \n",
       "political_conservative_US_gpt4T_dum                                    1.000000   \n",
       "political_conservative_US_gpt4o_system_dum                             0.685075   \n",
       "political_conservative_US_gpt4T_system_dum                             0.788238   \n",
       "\n",
       "                                            political_conservative_US_gpt4o_system_dum  \\\n",
       "political_conservative_US_dum                                                 0.756985   \n",
       "political_conservative_US_small_dum                                           0.446311   \n",
       "political_conservative_US_gpt4o_dum                                           0.877906   \n",
       "political_conservative_US_gpt4T_dum                                           0.685075   \n",
       "political_conservative_US_gpt4o_system_dum                                    1.000000   \n",
       "political_conservative_US_gpt4T_system_dum                                    0.791994   \n",
       "\n",
       "                                            political_conservative_US_gpt4T_system_dum  \n",
       "political_conservative_US_dum                                                 0.777826  \n",
       "political_conservative_US_small_dum                                           0.444957  \n",
       "political_conservative_US_gpt4o_dum                                           0.799466  \n",
       "political_conservative_US_gpt4T_dum                                           0.788238  \n",
       "political_conservative_US_gpt4o_system_dum                                    0.791994  \n",
       "political_conservative_US_gpt4T_system_dum                                    1.000000  "
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#for conservative:\n",
    "# Get all column names that contain 'conservative'\n",
    "conservative_columns = [col for col in dataset_w_pred_anon.columns if 'political_conservative_US' in col and col.endswith('_dum')]\n",
    "#check correlation between these GLLM annotations:\n",
    "conservative_subset = dataset_w_pred_anon.loc[:, conservative_columns].dropna()\n",
    "conservative_subset.corr()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6059f2bd",
   "metadata": {},
   "outputs": [],
   "source": [
    "#correlations among diversity similar to incivility, except for the small model, which has a lower correlation with the other models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "7440a9dd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['incivility_allcaps_dum', 'incivility_attackreputation_dum', 'incivility_question_intelligence_dum', 'incivility_sarcasm_dum', 'incivility_vulgarity_dum', 'incivility_namecalling_dum', 'incivility_prompt_dum', 'incivility_namecalling_updated_dum', 'incivility_namecalling_edit_dum', 'incivility_combine_dum', 'incivility_simple_dum', 'incivility_simple2_dum', 'incivility_simple2_small_dum', 'incivility_simple2_gpt4o_dum', 'incivility_simple2_gpt4T_dum', 'incivility_simple2_gpt4o_system_dum', 'incivility_simple2_gpt4T_system_dum']\n"
     ]
    }
   ],
   "source": [
    "#and if we vary the prompt:\n",
    "# Get all column names that contain 'incivility'\n",
    "incivility_columns = [col for col in dataset_w_pred_anon.columns if 'incivility' in col and col.endswith('_dum')]\n",
    "print(incivility_columns)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "c66fe60d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>incivility_combine_dum</th>\n",
       "      <th>incivility_simple_dum</th>\n",
       "      <th>incivility_simple2_dum</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>incivility_combine_dum</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.723273</td>\n",
       "      <td>0.756055</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>incivility_simple_dum</th>\n",
       "      <td>0.723273</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.800904</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>incivility_simple2_dum</th>\n",
       "      <td>0.756055</td>\n",
       "      <td>0.800904</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        incivility_combine_dum  incivility_simple_dum  \\\n",
       "incivility_combine_dum                1.000000               0.723273   \n",
       "incivility_simple_dum                 0.723273               1.000000   \n",
       "incivility_simple2_dum                0.756055               0.800904   \n",
       "\n",
       "                        incivility_simple2_dum  \n",
       "incivility_combine_dum                0.756055  \n",
       "incivility_simple_dum                 0.800904  \n",
       "incivility_simple2_dum                1.000000  "
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset_w_pred_anon.loc[:, ['incivility_combine_dum', 'incivility_simple_dum', 'incivility_simple2_dum']].corr()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "5f2c3e55",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['rationality_external_evidence_dum',\n",
       " 'rationality_topic_relevance_dum',\n",
       " 'rationality_background_info_low_dum',\n",
       " 'rationality_reasoning_dum',\n",
       " 'rationality_background_info_updated_dum',\n",
       " 'rationality_external_evidence_updated_dum',\n",
       " 'rationality_prompt_dum',\n",
       " 'rationality_reasoning_edit_dum',\n",
       " 'rationality_background_info_run2_dum',\n",
       " 'rationality_external_evidence_run2_dum',\n",
       " 'rationality_external_evidence_edit_dum',\n",
       " 'rationality_background_info_edit_dum',\n",
       " 'rationality_combine_dum',\n",
       " 'rationality_simple_dum',\n",
       " 'rationality_combine_exactexample_dum',\n",
       " 'rationality_simple2_dum',\n",
       " 'rationality_simple2_small_dum',\n",
       " 'rationality_simple2_gpt4o_dum',\n",
       " 'rationality_simple2_gpt4T_dum',\n",
       " 'rationality_simple2_gpt4o_system_dum',\n",
       " 'rationality_simple2_gpt4T_system_dum']"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get all column names that contain 'rationality'\n",
    "[col for col in dataset_w_pred_anon.columns if 'rationality' in col and col.endswith('_dum')]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "18e26998",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>rationality_combine_dum</th>\n",
       "      <th>rationality_simple_dum</th>\n",
       "      <th>rationality_combine_exactexample_dum</th>\n",
       "      <th>rationality_simple2_dum</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>rationality_combine_dum</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.541124</td>\n",
       "      <td>0.929029</td>\n",
       "      <td>0.422220</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rationality_simple_dum</th>\n",
       "      <td>0.541124</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.541646</td>\n",
       "      <td>0.666720</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rationality_combine_exactexample_dum</th>\n",
       "      <td>0.929029</td>\n",
       "      <td>0.541646</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.429706</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rationality_simple2_dum</th>\n",
       "      <td>0.422220</td>\n",
       "      <td>0.666720</td>\n",
       "      <td>0.429706</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      rationality_combine_dum  \\\n",
       "rationality_combine_dum                              1.000000   \n",
       "rationality_simple_dum                               0.541124   \n",
       "rationality_combine_exactexample_dum                 0.929029   \n",
       "rationality_simple2_dum                              0.422220   \n",
       "\n",
       "                                      rationality_simple_dum  \\\n",
       "rationality_combine_dum                             0.541124   \n",
       "rationality_simple_dum                              1.000000   \n",
       "rationality_combine_exactexample_dum                0.541646   \n",
       "rationality_simple2_dum                             0.666720   \n",
       "\n",
       "                                      rationality_combine_exactexample_dum  \\\n",
       "rationality_combine_dum                                           0.929029   \n",
       "rationality_simple_dum                                            0.541646   \n",
       "rationality_combine_exactexample_dum                              1.000000   \n",
       "rationality_simple2_dum                                           0.429706   \n",
       "\n",
       "                                      rationality_simple2_dum  \n",
       "rationality_combine_dum                              0.422220  \n",
       "rationality_simple_dum                               0.666720  \n",
       "rationality_combine_exactexample_dum                 0.429706  \n",
       "rationality_simple2_dum                              1.000000  "
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset_w_pred_anon.loc[:, ['rationality_combine_dum', 'rationality_simple_dum', 'rationality_combine_exactexample_dum', 'rationality_simple2_dum']].corr()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "098b6e9a",
   "metadata": {},
   "outputs": [],
   "source": [
    "#again much poorer correlation between the different GLLM annotations for rationality than for incivility"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "5d2e07bd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['interactivity_acknowledgement_dum',\n",
       " 'interactivity_acknowledgement_updated_dum',\n",
       " 'interactivity_acknowledgement_simple_dum',\n",
       " 'interactivity_acknowledgement_simple_small_dum',\n",
       " 'interactivity_acknowledgement_simple_small2_dum',\n",
       " 'interactivity_acknowledgement_simple2_dum',\n",
       " 'interactivity_acknowledgement_simple_gpt4o_dum',\n",
       " 'interactivity_acknowledgement_gpt4o_2_dum',\n",
       " 'interactivity_acknowledgement_simple_gpt4T_dum',\n",
       " 'interactivity_acknowledgement_simple_gpt4o_system_dum',\n",
       " 'interactivity_acknowledgement_simple_gpt4T_system_dum']"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get all column names that contain 'interactivity_acknowledgement'\n",
    "[col for col in dataset_w_pred_anon.columns if 'interactivity_acknowledgement' in col and col.endswith('_dum')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "f57ee035",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>interactivity_acknowledgement_dum</th>\n",
       "      <th>interactivity_acknowledgement_updated_dum</th>\n",
       "      <th>interactivity_acknowledgement_simple_dum</th>\n",
       "      <th>interactivity_acknowledgement_simple2_dum</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>interactivity_acknowledgement_dum</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.879747</td>\n",
       "      <td>0.718899</td>\n",
       "      <td>0.697970</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>interactivity_acknowledgement_updated_dum</th>\n",
       "      <td>0.879747</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.712814</td>\n",
       "      <td>0.703862</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>interactivity_acknowledgement_simple_dum</th>\n",
       "      <td>0.718899</td>\n",
       "      <td>0.712814</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.793577</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>interactivity_acknowledgement_simple2_dum</th>\n",
       "      <td>0.697970</td>\n",
       "      <td>0.703862</td>\n",
       "      <td>0.793577</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           interactivity_acknowledgement_dum  \\\n",
       "interactivity_acknowledgement_dum                                   1.000000   \n",
       "interactivity_acknowledgement_updated_dum                           0.879747   \n",
       "interactivity_acknowledgement_simple_dum                            0.718899   \n",
       "interactivity_acknowledgement_simple2_dum                           0.697970   \n",
       "\n",
       "                                           interactivity_acknowledgement_updated_dum  \\\n",
       "interactivity_acknowledgement_dum                                           0.879747   \n",
       "interactivity_acknowledgement_updated_dum                                   1.000000   \n",
       "interactivity_acknowledgement_simple_dum                                    0.712814   \n",
       "interactivity_acknowledgement_simple2_dum                                   0.703862   \n",
       "\n",
       "                                           interactivity_acknowledgement_simple_dum  \\\n",
       "interactivity_acknowledgement_dum                                          0.718899   \n",
       "interactivity_acknowledgement_updated_dum                                  0.712814   \n",
       "interactivity_acknowledgement_simple_dum                                   1.000000   \n",
       "interactivity_acknowledgement_simple2_dum                                  0.793577   \n",
       "\n",
       "                                           interactivity_acknowledgement_simple2_dum  \n",
       "interactivity_acknowledgement_dum                                           0.697970  \n",
       "interactivity_acknowledgement_updated_dum                                   0.703862  \n",
       "interactivity_acknowledgement_simple_dum                                    0.793577  \n",
       "interactivity_acknowledgement_simple2_dum                                   1.000000  "
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset_w_pred_anon.loc[:, ['interactivity_acknowledgement_dum', 'interactivity_acknowledgement_updated_dum', 'interactivity_acknowledgement_simple_dum', 'interactivity_acknowledgement_simple2_dum']].corr()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "22c85932",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['political_liberal_US_dum',\n",
       " 'diversity_liberal_dum',\n",
       " 'diversity_liberal_long_dum',\n",
       " 'political_liberal_US_small_dum',\n",
       " 'political_liberal_US_gpt4o_dum',\n",
       " 'political_liberal_US_gpt4T_dum',\n",
       " 'political_liberal_US_gpt4o_system_dum',\n",
       " 'political_liberal_US_gpt4T_system_dum']"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get all column names that contain 'liberal'\n",
    "[col for col in dataset_w_pred_anon.columns if 'liberal' in col and col.endswith('_dum')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "56a7983d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>political_liberal_US_dum</th>\n",
       "      <th>diversity_liberal_dum</th>\n",
       "      <th>diversity_liberal_long_dum</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>political_liberal_US_dum</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.644910</td>\n",
       "      <td>0.645108</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>diversity_liberal_dum</th>\n",
       "      <td>0.644910</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.938458</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>diversity_liberal_long_dum</th>\n",
       "      <td>0.645108</td>\n",
       "      <td>0.938458</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                            political_liberal_US_dum  diversity_liberal_dum  \\\n",
       "political_liberal_US_dum                    1.000000               0.644910   \n",
       "diversity_liberal_dum                       0.644910               1.000000   \n",
       "diversity_liberal_long_dum                  0.645108               0.938458   \n",
       "\n",
       "                            diversity_liberal_long_dum  \n",
       "political_liberal_US_dum                      0.645108  \n",
       "diversity_liberal_dum                         0.938458  \n",
       "diversity_liberal_long_dum                    1.000000  "
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset_w_pred_anon.loc[:, ['political_liberal_US_dum', 'diversity_liberal_dum', 'diversity_liberal_long_dum']].corr()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "aface7be",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['political_conservative_US_dum',\n",
       " 'diversity_conservative_dum',\n",
       " 'diversity_conservative_long_dum',\n",
       " 'political_conservative_US_small_dum',\n",
       " 'political_conservative_US_gpt4o_dum',\n",
       " 'political_conservative_US_gpt4T_dum',\n",
       " 'political_conservative_US_gpt4o_system_dum',\n",
       " 'political_conservative_US_gpt4T_system_dum']"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get all column names that contain 'conservative'\n",
    "[col for col in dataset_w_pred_anon.columns if 'conservative' in col and col.endswith('_dum')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "726aa422",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>political_conservative_US_dum</th>\n",
       "      <th>diversity_conservative_dum</th>\n",
       "      <th>diversity_conservative_long_dum</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>political_conservative_US_dum</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.599804</td>\n",
       "      <td>0.613843</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>diversity_conservative_dum</th>\n",
       "      <td>0.599804</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.914279</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>diversity_conservative_long_dum</th>\n",
       "      <td>0.613843</td>\n",
       "      <td>0.914279</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                 political_conservative_US_dum  \\\n",
       "political_conservative_US_dum                         1.000000   \n",
       "diversity_conservative_dum                            0.599804   \n",
       "diversity_conservative_long_dum                       0.613843   \n",
       "\n",
       "                                 diversity_conservative_dum  \\\n",
       "political_conservative_US_dum                      0.599804   \n",
       "diversity_conservative_dum                         1.000000   \n",
       "diversity_conservative_long_dum                    0.914279   \n",
       "\n",
       "                                 diversity_conservative_long_dum  \n",
       "political_conservative_US_dum                           0.613843  \n",
       "diversity_conservative_dum                              0.914279  \n",
       "diversity_conservative_long_dum                         1.000000  "
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset_w_pred_anon.loc[:, ['political_conservative_US_dum', 'diversity_conservative_dum', 'diversity_conservative_long_dum']].corr()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0db00a43",
   "metadata": {},
   "outputs": [],
   "source": [
    "#different prompts can lead to correlations of similar levels as between different GLLMs"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "llmdiv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}