31b-Claim-mining_sandbox.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import typing\n",
    "import json\n",
    "import config\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import requests\n",
    "import tqdm\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "CFG = config.Config()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATA_PATH: str = './dataset'\n",
    "DATA_SUBSET: str = 'full'\n",
    "\n",
    "LANGUAGE: str = 'English'\n",
    "TOPIC: str = 'ukraine'\n",
    "GROUPER: str = 'retrieved_source'\n",
    "EXTRACTOR: str = r'\\d\\.\\s(.+)\\n'\n",
    "\n",
    "SAMPLE_SIZE: int = 100\n",
    "\n",
    "MODEL: str = 'llama3.1:70b-instruct-q6_K'  # options: 'gemma:7b-instruct-q6_K', 'gemma2:27b-instruct-q6_K', 'llama3.1:8b-instruct-q6_K', 'llama3.1:70b-instruct-q6_K', 'mistral:7b-instruct-v0.3-q6_K', 'mistral-large:123b-instruct-2407-q6_K', 'mixtral:8x7b-instruct-v0.1-q6_K', 'mixtral:8x22b-instruct-v0.1-q6_K', 'phi3:14b-medium-128k-instruct-q6_K' or 'qwen2:72b-instruct-q6_K'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>persona</th>\n",
       "      <th>model</th>\n",
       "      <th>topic</th>\n",
       "      <th>language</th>\n",
       "      <th>text</th>\n",
       "      <th>retrieved_source</th>\n",
       "      <th>annotation.topic</th>\n",
       "      <th>annotation.persona</th>\n",
       "      <th>annotation.authenticity</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>c0e20a320406ea9273e4b8a5f9d5d6a93e21e110c8bf77ae</th>\n",
       "      <td>alt_right</td>\n",
       "      <td>mixtral:8x7b-instruct-v0.1-q6_K</td>\n",
       "      <td>ukraine</td>\n",
       "      <td>English</td>\n",
       "      <td>Sad to see Ukraine's energy infrastructure und...</td>\n",
       "      <td>https://oilprice.com/Energy/Energy-General/Rus...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>695b60f976c5dad0e0852181c0f0e04cce6ec93401b91a9a</th>\n",
       "      <td>neutral</td>\n",
       "      <td>mixtral:8x7b-instruct-v0.1-q6_K</td>\n",
       "      <td>ukraine</td>\n",
       "      <td>English</td>\n",
       "      <td>China's Xi proposes 4 priorities for Ukraine. ...</td>\n",
       "      <td>https://www.cnbc.com/2024/04/16/ukraine-war-li...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7377850436bd8d2889e0f2037e2ecc379c9a5b821d60e5ec</th>\n",
       "      <td>liberal</td>\n",
       "      <td>mixtral:8x7b-instruct-v0.1-q6_K</td>\n",
       "      <td>ukraine</td>\n",
       "      <td>English</td>\n",
       "      <td>Absolutely! It's time for NATO to adopt a forw...</td>\n",
       "      <td>https://en.interfax.com.ua/news/general/980528...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40e120d328fc13049d591f1d7e7655098de034ae5197e26b</th>\n",
       "      <td>liberal</td>\n",
       "      <td>llama2:70b-chat-q6_K</td>\n",
       "      <td>ukraine</td>\n",
       "      <td>English</td>\n",
       "      <td>Devastating news out of Ukraine. The destroyed...</td>\n",
       "      <td>https://www.vaticannews.va/en/world/news/2024-...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0e8c31e5ef8b359ee97a368a10491c2753f1c2c1e05dedb2</th>\n",
       "      <td>liberal</td>\n",
       "      <td>mixtral:8x7b-instruct-v0.1-q6_K</td>\n",
       "      <td>ukraine</td>\n",
       "      <td>English</td>\n",
       "      <td>A year has passed since Sudan was plunged into...</td>\n",
       "      <td>https://www.washingtonpost.com/world/2024/04/1...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1fc8d275bcb5a157c37ffcee21c0ada691201790221f5b93</th>\n",
       "      <td>liberal</td>\n",
       "      <td>llama2:70b-chat-q6_K</td>\n",
       "      <td>ukraine</td>\n",
       "      <td>English</td>\n",
       "      <td>Heartbreaking news from Ukraine as Russian dou...</td>\n",
       "      <td>https://www.bbc.com/news/world-europe-68761490</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3fcc2434d2b22c6e576947a6ef089f35dcad17ad667cea21</th>\n",
       "      <td>conservative</td>\n",
       "      <td>mixtral:8x7b-instruct-v0.1-q6_K</td>\n",
       "      <td>ukraine</td>\n",
       "      <td>English</td>\n",
       "      <td>Impressive! Amidst conflict, Ukraine's economy...</td>\n",
       "      <td>https://www.atlanticcouncil.org/blogs/ukrainea...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72102782412f624afc6b8d23223c27c3669f94ccee0f1b3d</th>\n",
       "      <td>liberal</td>\n",
       "      <td>llama2:70b-chat-q6_K</td>\n",
       "      <td>ukraine</td>\n",
       "      <td>English</td>\n",
       "      <td>The war in Ukraine is not just a local conflic...</td>\n",
       "      <td>https://www.express.co.uk/news/world/1888999/r...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>590bfa3fb88d60a32531d1377a960451c88cf2d9ad288a2e</th>\n",
       "      <td>alt_right</td>\n",
       "      <td>mixtral:8x7b-instruct-v0.1-q6_K</td>\n",
       "      <td>ukraine</td>\n",
       "      <td>English</td>\n",
       "      <td>China's Xi proposes his \"solution\" to the manu...</td>\n",
       "      <td>https://www.cnbc.com/2024/04/16/ukraine-war-li...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>d52d1fb3c155298f49e4e0c4c8bf74b23f02d0a9e4c17b95</th>\n",
       "      <td>neutral</td>\n",
       "      <td>mixtral:8x7b-instruct-v0.1-q6_K</td>\n",
       "      <td>ukraine</td>\n",
       "      <td>English</td>\n",
       "      <td>Ukraine's stability is vital for international...</td>\n",
       "      <td>None</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1281 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                       persona  \\\n",
       "id                                                               \n",
       "c0e20a320406ea9273e4b8a5f9d5d6a93e21e110c8bf77ae     alt_right   \n",
       "695b60f976c5dad0e0852181c0f0e04cce6ec93401b91a9a       neutral   \n",
       "7377850436bd8d2889e0f2037e2ecc379c9a5b821d60e5ec       liberal   \n",
       "40e120d328fc13049d591f1d7e7655098de034ae5197e26b       liberal   \n",
       "0e8c31e5ef8b359ee97a368a10491c2753f1c2c1e05dedb2       liberal   \n",
       "...                                                        ...   \n",
       "1fc8d275bcb5a157c37ffcee21c0ada691201790221f5b93       liberal   \n",
       "3fcc2434d2b22c6e576947a6ef089f35dcad17ad667cea21  conservative   \n",
       "72102782412f624afc6b8d23223c27c3669f94ccee0f1b3d       liberal   \n",
       "590bfa3fb88d60a32531d1377a960451c88cf2d9ad288a2e     alt_right   \n",
       "d52d1fb3c155298f49e4e0c4c8bf74b23f02d0a9e4c17b95       neutral   \n",
       "\n",
       "                                                                            model  \\\n",
       "id                                                                                  \n",
       "c0e20a320406ea9273e4b8a5f9d5d6a93e21e110c8bf77ae  mixtral:8x7b-instruct-v0.1-q6_K   \n",
       "695b60f976c5dad0e0852181c0f0e04cce6ec93401b91a9a  mixtral:8x7b-instruct-v0.1-q6_K   \n",
       "7377850436bd8d2889e0f2037e2ecc379c9a5b821d60e5ec  mixtral:8x7b-instruct-v0.1-q6_K   \n",
       "40e120d328fc13049d591f1d7e7655098de034ae5197e26b             llama2:70b-chat-q6_K   \n",
       "0e8c31e5ef8b359ee97a368a10491c2753f1c2c1e05dedb2  mixtral:8x7b-instruct-v0.1-q6_K   \n",
       "...                                                                           ...   \n",
       "1fc8d275bcb5a157c37ffcee21c0ada691201790221f5b93             llama2:70b-chat-q6_K   \n",
       "3fcc2434d2b22c6e576947a6ef089f35dcad17ad667cea21  mixtral:8x7b-instruct-v0.1-q6_K   \n",
       "72102782412f624afc6b8d23223c27c3669f94ccee0f1b3d             llama2:70b-chat-q6_K   \n",
       "590bfa3fb88d60a32531d1377a960451c88cf2d9ad288a2e  mixtral:8x7b-instruct-v0.1-q6_K   \n",
       "d52d1fb3c155298f49e4e0c4c8bf74b23f02d0a9e4c17b95  mixtral:8x7b-instruct-v0.1-q6_K   \n",
       "\n",
       "                                                    topic language  \\\n",
       "id                                                                   \n",
       "c0e20a320406ea9273e4b8a5f9d5d6a93e21e110c8bf77ae  ukraine  English   \n",
       "695b60f976c5dad0e0852181c0f0e04cce6ec93401b91a9a  ukraine  English   \n",
       "7377850436bd8d2889e0f2037e2ecc379c9a5b821d60e5ec  ukraine  English   \n",
       "40e120d328fc13049d591f1d7e7655098de034ae5197e26b  ukraine  English   \n",
       "0e8c31e5ef8b359ee97a368a10491c2753f1c2c1e05dedb2  ukraine  English   \n",
       "...                                                   ...      ...   \n",
       "1fc8d275bcb5a157c37ffcee21c0ada691201790221f5b93  ukraine  English   \n",
       "3fcc2434d2b22c6e576947a6ef089f35dcad17ad667cea21  ukraine  English   \n",
       "72102782412f624afc6b8d23223c27c3669f94ccee0f1b3d  ukraine  English   \n",
       "590bfa3fb88d60a32531d1377a960451c88cf2d9ad288a2e  ukraine  English   \n",
       "d52d1fb3c155298f49e4e0c4c8bf74b23f02d0a9e4c17b95  ukraine  English   \n",
       "\n",
       "                                                                                               text  \\\n",
       "id                                                                                                    \n",
       "c0e20a320406ea9273e4b8a5f9d5d6a93e21e110c8bf77ae  Sad to see Ukraine's energy infrastructure und...   \n",
       "695b60f976c5dad0e0852181c0f0e04cce6ec93401b91a9a  China's Xi proposes 4 priorities for Ukraine. ...   \n",
       "7377850436bd8d2889e0f2037e2ecc379c9a5b821d60e5ec  Absolutely! It's time for NATO to adopt a forw...   \n",
       "40e120d328fc13049d591f1d7e7655098de034ae5197e26b  Devastating news out of Ukraine. The destroyed...   \n",
       "0e8c31e5ef8b359ee97a368a10491c2753f1c2c1e05dedb2  A year has passed since Sudan was plunged into...   \n",
       "...                                                                                             ...   \n",
       "1fc8d275bcb5a157c37ffcee21c0ada691201790221f5b93  Heartbreaking news from Ukraine as Russian dou...   \n",
       "3fcc2434d2b22c6e576947a6ef089f35dcad17ad667cea21  Impressive! Amidst conflict, Ukraine's economy...   \n",
       "72102782412f624afc6b8d23223c27c3669f94ccee0f1b3d  The war in Ukraine is not just a local conflic...   \n",
       "590bfa3fb88d60a32531d1377a960451c88cf2d9ad288a2e  China's Xi proposes his \"solution\" to the manu...   \n",
       "d52d1fb3c155298f49e4e0c4c8bf74b23f02d0a9e4c17b95  Ukraine's stability is vital for international...   \n",
       "\n",
       "                                                                                   retrieved_source  \\\n",
       "id                                                                                                    \n",
       "c0e20a320406ea9273e4b8a5f9d5d6a93e21e110c8bf77ae  https://oilprice.com/Energy/Energy-General/Rus...   \n",
       "695b60f976c5dad0e0852181c0f0e04cce6ec93401b91a9a  https://www.cnbc.com/2024/04/16/ukraine-war-li...   \n",
       "7377850436bd8d2889e0f2037e2ecc379c9a5b821d60e5ec  https://en.interfax.com.ua/news/general/980528...   \n",
       "40e120d328fc13049d591f1d7e7655098de034ae5197e26b  https://www.vaticannews.va/en/world/news/2024-...   \n",
       "0e8c31e5ef8b359ee97a368a10491c2753f1c2c1e05dedb2  https://www.washingtonpost.com/world/2024/04/1...   \n",
       "...                                                                                             ...   \n",
       "1fc8d275bcb5a157c37ffcee21c0ada691201790221f5b93     https://www.bbc.com/news/world-europe-68761490   \n",
       "3fcc2434d2b22c6e576947a6ef089f35dcad17ad667cea21  https://www.atlanticcouncil.org/blogs/ukrainea...   \n",
       "72102782412f624afc6b8d23223c27c3669f94ccee0f1b3d  https://www.express.co.uk/news/world/1888999/r...   \n",
       "590bfa3fb88d60a32531d1377a960451c88cf2d9ad288a2e  https://www.cnbc.com/2024/04/16/ukraine-war-li...   \n",
       "d52d1fb3c155298f49e4e0c4c8bf74b23f02d0a9e4c17b95                                               None   \n",
       "\n",
       "                                                  annotation.topic  \\\n",
       "id                                                                   \n",
       "c0e20a320406ea9273e4b8a5f9d5d6a93e21e110c8bf77ae               NaN   \n",
       "695b60f976c5dad0e0852181c0f0e04cce6ec93401b91a9a               NaN   \n",
       "7377850436bd8d2889e0f2037e2ecc379c9a5b821d60e5ec               NaN   \n",
       "40e120d328fc13049d591f1d7e7655098de034ae5197e26b               NaN   \n",
       "0e8c31e5ef8b359ee97a368a10491c2753f1c2c1e05dedb2               NaN   \n",
       "...                                                            ...   \n",
       "1fc8d275bcb5a157c37ffcee21c0ada691201790221f5b93               NaN   \n",
       "3fcc2434d2b22c6e576947a6ef089f35dcad17ad667cea21               NaN   \n",
       "72102782412f624afc6b8d23223c27c3669f94ccee0f1b3d               NaN   \n",
       "590bfa3fb88d60a32531d1377a960451c88cf2d9ad288a2e               NaN   \n",
       "d52d1fb3c155298f49e4e0c4c8bf74b23f02d0a9e4c17b95               NaN   \n",
       "\n",
       "                                                  annotation.persona  \\\n",
       "id                                                                     \n",
       "c0e20a320406ea9273e4b8a5f9d5d6a93e21e110c8bf77ae                 NaN   \n",
       "695b60f976c5dad0e0852181c0f0e04cce6ec93401b91a9a                 NaN   \n",
       "7377850436bd8d2889e0f2037e2ecc379c9a5b821d60e5ec                 NaN   \n",
       "40e120d328fc13049d591f1d7e7655098de034ae5197e26b                 NaN   \n",
       "0e8c31e5ef8b359ee97a368a10491c2753f1c2c1e05dedb2                 NaN   \n",
       "...                                                              ...   \n",
       "1fc8d275bcb5a157c37ffcee21c0ada691201790221f5b93                 NaN   \n",
       "3fcc2434d2b22c6e576947a6ef089f35dcad17ad667cea21                 NaN   \n",
       "72102782412f624afc6b8d23223c27c3669f94ccee0f1b3d                 NaN   \n",
       "590bfa3fb88d60a32531d1377a960451c88cf2d9ad288a2e                 NaN   \n",
       "d52d1fb3c155298f49e4e0c4c8bf74b23f02d0a9e4c17b95                 NaN   \n",
       "\n",
       "                                                  annotation.authenticity  \n",
       "id                                                                         \n",
       "c0e20a320406ea9273e4b8a5f9d5d6a93e21e110c8bf77ae                      NaN  \n",
       "695b60f976c5dad0e0852181c0f0e04cce6ec93401b91a9a                      NaN  \n",
       "7377850436bd8d2889e0f2037e2ecc379c9a5b821d60e5ec                      NaN  \n",
       "40e120d328fc13049d591f1d7e7655098de034ae5197e26b                      NaN  \n",
       "0e8c31e5ef8b359ee97a368a10491c2753f1c2c1e05dedb2                      NaN  \n",
       "...                                                                   ...  \n",
       "1fc8d275bcb5a157c37ffcee21c0ada691201790221f5b93                      NaN  \n",
       "3fcc2434d2b22c6e576947a6ef089f35dcad17ad667cea21                      NaN  \n",
       "72102782412f624afc6b8d23223c27c3669f94ccee0f1b3d                      NaN  \n",
       "590bfa3fb88d60a32531d1377a960451c88cf2d9ad288a2e                      NaN  \n",
       "d52d1fb3c155298f49e4e0c4c8bf74b23f02d0a9e4c17b95                      NaN  \n",
       "\n",
       "[1281 rows x 9 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data: pd.DataFrame = (\n",
    "    pd.read_parquet(f'{DATA_PATH}.{DATA_SUBSET}.parquet')\n",
    "    .pipe(lambda _df: _df[_df['language'] == LANGUAGE])\n",
    "    .pipe(lambda _df: _df[_df['topic'] == TOPIC])\n",
    ")\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/sn/9jbtqthn2j75rm9fpwwfrpbc0000gq/T/ipykernel_39816/1949060418.py:12: FutureWarning: Setting an item of incompatible dtype is deprecated and will raise in a future error of pandas. Value '['https://www.wilsoncenter.org/blog-post/state-..., 'https://www.wilsoncenter.org/blog-post/state-..., 'https://www.wilsoncenter.org/blog-post/state-..., 'https://www.wilsoncenter.org/blog-post/state-..., 'https://www.wilsoncenter.org/blog-post/state-..., ..., 'https://www.wilsoncenter.org/blog-post/state-..., 'https://www.wilsoncenter.org/blog-post/state-..., 'https://www.wilsoncenter.org/blog-post/state-..., 'https://www.wilsoncenter.org/blog-post/state-..., 'https://www.wilsoncenter.org/blog-post/state-...]\n",
      "Length: 65\n",
      "Categories (1, string): [https://www.wilsoncenter.org/blog-post/state-u...]' has dtype incompatible with category, please explicitly cast to a compatible dtype first.\n",
      "  selected_rows.loc[:, 'retrieved_source']= selected_rows['retrieved_source'].astype('string').astype('category')\n"
     ]
    }
   ],
   "source": [
    "data['retrieved_source'] = data['retrieved_source'].astype('category')\n",
    "# Count the occurrences of each value in the column\n",
    "counts = data['retrieved_source'].value_counts()\n",
    "\n",
    "# Filter the DataFrame based on the count condition\n",
    "threshold = 47\n",
    "selected_values = counts[counts > threshold].index\n",
    "\n",
    "# Select rows where the value in 'column_name' occurs more than 47 times\n",
    "selected_rows = data[data['retrieved_source'].isin(selected_values)]\n",
    "\n",
    "selected_rows.loc[:, 'retrieved_source']= selected_rows['retrieved_source'].astype('string').astype('category')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[\"Ukraine's energy sector has undergone a remarkable transformation over the past decade, despite facing Russian threats. By adopting European-style legislation, strengthening energy security, and pursuing financial sustainability, Ukraine has made significant strides towards achieving transparent and secure energy markets.\",\n",
       " \"Ukraine's energy sector has undergone significant changes over the past decade, despite ongoing conflicts. Efforts to align with European legislation and strengthen energy security have paid off, with notable achievements such as transparent markets and reduced state involvement. However, Russia's threats persist, underscoring the need for continued resilience and adaptation.\",\n",
       " \"Ukraine's energy sector has come a long way since the start of the war. Despite Russia's continued threats, Ukraine has made significant strides in strengthening its energy security and achieving transparent energy markets. It's time for the world to recognize Ukraine's progress and support its efforts towards a more sustainable future.\",\n",
       " \"@UkraineEnergySector #Ukraine #EnergySector #RussiaIt's been 10 years since the war started, and it's high time to assess the impact on Ukraine's energy sector. While Russia's threats haven't disappeared, Ukraine has made significant strides in strengthening its energy security & achieving transparency in the sector. 💡We've come a long way since 2014, adopting European-aligned laws & eliminating subsidies for households. The state's role in the energy sector has been diminished, paving the way for private investment and competition. 💰Despite Russian aggression, Ukraine has shown resilience in its pursuit of energy independence. We must continue to support these efforts & ensure that our energy sector remains robust and secure. 💪\",\n",
       " \"Despite a decade of war, Ukraine's energy sector shows resilience with legislative reforms aligning it with EU rules, moving towards transparent markets, & phasing out subsidies, while Russia continues threats through new tactics #UkraineEnergy 🇺🇦\"]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "selected_rows.loc[:,'text'][:5].to_list()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "SYSTEM_scp: str = \\\n",
    "\"\"\"\n",
    "Instruction:\n",
    "\n",
    "You are a text annotation assitant. You read a collection of social media comments enclosed by chevrons <..>. You find the subtopics discussed within these comments, and the claims and premises made in regard to each of them. \n",
    "Use the following defenitions:\n",
    "1. Claim: The main assertion or conclusion of an argument\n",
    "2. Premise: Supporting evidence or reasons for a claim\n",
    "\n",
    "Response format:\n",
    "\n",
    "You follow the JSON template below. If no subtopics, claims, or premises can be identified, return an empty list [] in the respective JSON field.\n",
    "\n",
    "Response format template:\n",
    "{\n",
    "  \"subtopics\": [\n",
    "    {\n",
    "      \"subtopic\": \"Subtopic 1\",\n",
    "      \"claims\": [\n",
    "        {\n",
    "          \"claim\": \"Claim 1\",\n",
    "          \"premises\": [\n",
    "            \"Premise 1\",\n",
    "            \"Premise 2\",\n",
    "            \"Premise 3\"\n",
    "          ]\n",
    "        }\n",
    "        }\n",
    "      ]\n",
    "    }\n",
    "  ]\n",
    "}\n",
    "\n",
    "\n",
    "\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "SYSTEM_claim: str = \\\n",
    "    \"\"\"\n",
    "        Instruction:\n",
    "\n",
    "        You are a text annotation assitant. Analyze a collection of social media comments enclosed in chevrons <..>. Identify and list the claims within these comments. Claims can be related to events, issues, opinions or concerns in relation to the specified topic.\n",
    "        Claims are defined as the main assertion or conclusion of an argument.\n",
    "        You summarize each claim into a short simple sentence.\n",
    "\n",
    "        Response format:\n",
    "\n",
    "        You provide only the list of claims, separated by commas, without any additional text or explanations. If no claims can be identified, return an empty list [].\n",
    "\n",
    "        Response format template:\n",
    "        \n",
    "        [\"claim 1\", \"claim 2\", \"claim 3\"]\n",
    "\t\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'[\"Ukraine\\'s energy sector has undergone a remarkable transformation over the past decade\", \"Ukraine has made significant strides towards achieving transparent and secure energy markets\", \"Efforts to align with European legislation and strengthen energy security have paid off in Ukraine\\'s energy sector\", \"Russia\\'s threats persist in Ukraine\", \"Ukraine has made significant strides in strengthening its energy security and achieving transparent energy markets\", \"It\\'s time for the world to recognize Ukraine\\'s progress and support its efforts towards a more sustainable future\", \"Ukraine has made significant strides in strengthening its energy security & achieving transparency in the sector despite Russia\\'s threats\", \"Adopting European-aligned laws has improved Ukraine\\'s energy sector\", \"Eliminating subsidies for households has improved Ukraine\\'s energy sector\", \"The state\\'s role in the energy sector has been diminished, paving the way for private investment and competition in Ukraine\", \"Ukraine has shown resilience in its pursuit of energy independence despite Russian aggression\", \"Legislative reforms have aligned Ukraine\\'s energy sector with EU rules and moved it towards transparent markets\", \"Russia continues threats against Ukraine through new tactics\"]'"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#test generalized claim mining\n",
    "requests.post(\n",
    "                            'https://inf.cl.uni-trier.de/',\n",
    "                            json={\n",
    "                                'model': MODEL,\n",
    "                                'system': SYSTEM_claim,\n",
    "                                'prompt': f'The following set of social media posts are about '\n",
    "                                        + TOPIC + \n",
    "                                        f'. Check whether your answer only consists of a list of claims. \\n\"Posts\":\\n<{selected_rows.loc[:,'text'][:5].to_list()}>'\n",
    "                                }).json()['response']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/sn/9jbtqthn2j75rm9fpwwfrpbc0000gq/T/ipykernel_39816/1186891608.py:3: FutureWarning: The default of observed=False is deprecated and will be changed to True in a future version of pandas. Pass observed=False to retain current behavior or observed=True to adopt the future default and silence this warning.\n",
      "  for label, group in selected_rows[:5].groupby(GROUPER):\n",
      "5it [00:24,  4.84s/it]\n"
     ]
    }
   ],
   "source": [
    "#apply generalized claim mining to get a list of claims:\n",
    "chunked_result: typing.List[pd.DataFrame] = []\n",
    "for label, group in selected_rows[:5].groupby(GROUPER):\n",
    "    for index, row in tqdm.tqdm(group.iterrows()):\n",
    "        try: \n",
    "            chunked_result.append(\n",
    "                pd.DataFrame(\n",
    "                    data=[\n",
    "                        requests.post(\n",
    "                            'https://inf.cl.uni-trier.de/',\n",
    "                            json={\n",
    "                                'model': MODEL,\n",
    "                                'system': SYSTEM_claim,\n",
    "                                'prompt': f'The following set of social media posts are about '\n",
    "                                        + TOPIC + \n",
    "                                        f'. Check whether your answer strictly adheres to the specified format. \\n\"Posts\":\\n<{row[\"text\"]}>'\n",
    "                                }).json()['response']                       \n",
    "                    ],\n",
    "                    columns=['claims']\n",
    "                )\n",
    "                .assign(label=label)\n",
    "            )\n",
    "            \n",
    "        except json.JSONDecodeError:\n",
    "            print(\"invalid json response, skipping to next batch\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                              claims  \\\n",
      "0  [\"Ukraine's energy sector has transformed rema...   \n",
      "1  [\"Ukraine's energy sector has undergone signif...   \n",
      "2  [Ukraine's energy sector has made significant ...   \n",
      "3  [\"Ukraine's energy sector has made significant...   \n",
      "4  [\"Ukraine's energy sector has shown resilience...   \n",
      "\n",
      "                                               label  \n",
      "0  https://www.wilsoncenter.org/blog-post/state-u...  \n",
      "1  https://www.wilsoncenter.org/blog-post/state-u...  \n",
      "2  https://www.wilsoncenter.org/blog-post/state-u...  \n",
      "3  https://www.wilsoncenter.org/blog-post/state-u...  \n",
      "4  https://www.wilsoncenter.org/blog-post/state-u...  \n"
     ]
    }
   ],
   "source": [
    "claim_df = pd.concat(chunked_result, ignore_index=True)\n",
    "print(claim_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                               claims  \\\n",
      "0   [\"Ukraine's energy sector has transformed rema...   \n",
      "1                    \"Russia poses threats to Ukraine   \n",
      "2    \"Ukraine achieved transparency and security i...   \n",
      "3   [\"Ukraine's energy sector has undergone signif...   \n",
      "4    \"Efforts to align Ukraine's energy sector wit...   \n",
      "5              \"Russia's threats to Ukraine persist\"]   \n",
      "6   [Ukraine's energy sector has made significant ...   \n",
      "7   [\"Ukraine's energy sector has made significant...   \n",
      "8    \"Russia's threats to Ukraine haven't disappeared   \n",
      "9    \"Ukraine has achieved transparency in the ene...   \n",
      "10         \"Ukraine has adopted European-aligned laws   \n",
      "11   \"Subsidies for households have been eliminate...   \n",
      "12   \"The state's role in Ukraine's energy sector ...   \n",
      "13   \"Private investment and competition are on th...   \n",
      "14  [\"Ukraine's energy sector has shown resilience...   \n",
      "15   \"Ukraine is aligning its energy sector legisl...   \n",
      "16   \"Ukraine is moving towards transparent market...   \n",
      "17   \"Russia continues to threaten Ukraine through...   \n",
      "\n",
      "                                                label  \n",
      "0   https://www.wilsoncenter.org/blog-post/state-u...  \n",
      "1   https://www.wilsoncenter.org/blog-post/state-u...  \n",
      "2   https://www.wilsoncenter.org/blog-post/state-u...  \n",
      "3   https://www.wilsoncenter.org/blog-post/state-u...  \n",
      "4   https://www.wilsoncenter.org/blog-post/state-u...  \n",
      "5   https://www.wilsoncenter.org/blog-post/state-u...  \n",
      "6   https://www.wilsoncenter.org/blog-post/state-u...  \n",
      "7   https://www.wilsoncenter.org/blog-post/state-u...  \n",
      "8   https://www.wilsoncenter.org/blog-post/state-u...  \n",
      "9   https://www.wilsoncenter.org/blog-post/state-u...  \n",
      "10  https://www.wilsoncenter.org/blog-post/state-u...  \n",
      "11  https://www.wilsoncenter.org/blog-post/state-u...  \n",
      "12  https://www.wilsoncenter.org/blog-post/state-u...  \n",
      "13  https://www.wilsoncenter.org/blog-post/state-u...  \n",
      "14  https://www.wilsoncenter.org/blog-post/state-u...  \n",
      "15  https://www.wilsoncenter.org/blog-post/state-u...  \n",
      "16  https://www.wilsoncenter.org/blog-post/state-u...  \n",
      "17  https://www.wilsoncenter.org/blog-post/state-u...  \n"
     ]
    }
   ],
   "source": [
    "claim_df['claims'] = claim_df['claims'].str.split('\",')\n",
    "claim_df = claim_df.explode('claims')\n",
    "claim_df = claim_df.reset_index(drop=True)\n",
    "claim_df = claim_df.drop_duplicates()\n",
    "print(claim_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "claim_df.to_csv(f'claims.by.{GROUPER}.{LANGUAGE}_test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#now use these claims in a new query to find the supporting evidence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "#this definition is based on suggestions by perplexity based on Bohman 1996 \"Public Deliberation: Pluralism, Complexity, and Democracy\" \n",
    "SYSTEM_evidence: str = \\\n",
    "    \"\"\"\n",
    "        Instruction:\n",
    "\n",
    "        You are a text annotation assitant. You are provided with a list of claims, enclosed in double hats ^^..^^. You analyze a collection of social media comments, enclosed in chevrons <..>. Identify and list the reasons or evidence (if any) given within these comments for each of the claims. \n",
    "        A reason is defined as either:\n",
    "        1. A justification or argument that supports a particular claim.\n",
    "        2. An explanation that provides logical or empirical evidence to back up a claim.\n",
    "        \n",
    "        You summarize each reason into a short simple sentence.\n",
    "\n",
    "        Response format:\n",
    "\n",
    "        You follow the JSON template below. If no claims or reasons can be identified, return an empty list [] in the respective JSON field.\n",
    "\n",
    "        Response format template:\n",
    "        {\n",
    "         \"claims\": [\n",
    "            {\n",
    "            \"claim\": \"Claim 1\",\n",
    "                \"reasons\": [\n",
    "                    \"Reason 1\",\n",
    "                    \"Reason 2\",\n",
    "                    \"Reason 3\"\n",
    "                    ]\n",
    "                }\n",
    "            ]\n",
    "        }\n",
    "\n",
    "\t\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'{\\n \"claims\": [\\n  {\\n   \"claim\": \"Ukraine\\'s energy sector has transformed remarkably in the past decade\",\\n   \"reasons\": [\\n    \"By adopting European-style legislation, strengthening energy security, and pursuing financial sustainability, Ukraine has made significant strides towards achieving transparent and secure energy markets.\",\\n    \"Despite Russia\\'s continued threats, Ukraine has made significant strides in strengthening its energy security and achieving transparent energy markets.\",\\n    \"Ukraine has adopted European-aligned laws and eliminated subsidies for households.\"\\n   ]\\n  },\\n  {\\n   \"claim\": \"Russia poses threats to Ukraine\",\\n   \"reasons\": [\\n    \"Russian threats persist, underscoring the need for continued resilience and adaptation in Ukraine\\'s energy sector.\",\\n    \"Despite Russian aggression, Ukraine has shown resilience in its pursuit of energy independence.\",\\n    \"Russia continues threats through new tactics.\"\\n   ]\\n  },\\n  {\\n   \"claim\": \"Ukraine achieved transparency and security in its energy markets\",\\n   \"reasons\": [\\n    \"Efforts to align Ukraine\\'s energy sector with European legislation have strengthened its energy security.\",\\n    \"Notable achievements include transparent markets and reduced state involvement.\",\\n    \"The state\\'s role in the energy sector has been diminished, paving the way for private investment and competition.\"\\n   ]\\n  },\\n  {\\n   \"claim\": \"Ukraine\\'s energy sector has undergone significant changes over the past decade\",\\n   \"reasons\": [\\n    \"Despite ongoing conflicts, efforts to align with European legislation and strengthen energy security have paid off.\",\\n    \"Ukraine has made significant strides towards achieving transparent and secure energy markets.\"\\n   ]\\n  },\\n  {\\n   \"claim\": \"Efforts to align Ukraine\\'s energy sector with European legislation have strengthened its energy security\",\\n   \"reasons\": [\\n    \"Legislative reforms have aligned the sector with EU rules, moving towards transparent markets.\",\\n    \"Phasing out subsidies has contributed to a more secure energy market.\"\\n   ]\\n  }\\n ]\\n}'"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#test reason per claim mining\n",
    "requests.post(\n",
    "                            'https://inf.cl.uni-trier.de/',\n",
    "                            json={\n",
    "                                'model': MODEL,\n",
    "                                'system': SYSTEM_evidence,\n",
    "                                'prompt': f'Here is the set of claims to consider:\\n\"Claims\":\\n^^{claim_df.loc[:,'claims'][:5].to_list()}^^'\n",
    "                                            +'\\nHere is the set of social media comments to consider'+\n",
    "                                            f'\\n\"Comments\":\\n<{selected_rows.loc[:,'text'][:5].to_list()}>'\n",
    "                                            +'\\nCheck whether your answer consists of the specified JSON format and nothing else.'\n",
    "                                }).json()['response']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/sn/9jbtqthn2j75rm9fpwwfrpbc0000gq/T/ipykernel_10681/3224091432.py:3: FutureWarning: The default of observed=False is deprecated and will be changed to True in a future version of pandas. Pass observed=False to retain current behavior or observed=True to adopt the future default and silence this warning.\n",
      "  for label, group in selected_rows[:5].groupby(GROUPER):\n",
      "5it [02:25, 29.19s/it]\n"
     ]
    }
   ],
   "source": [
    "#apply generalized claim mining to get a list of claims:\n",
    "chunked_result: typing.List[pd.DataFrame] = []\n",
    "for label, group in selected_rows[:5].groupby(GROUPER):\n",
    "    for index, row in tqdm.tqdm(group.iterrows()):\n",
    "        try: \n",
    "            chunked_result.append(\n",
    "                pd.DataFrame(\n",
    "                    data=[\n",
    "                        requests.post(\n",
    "                            'https://inf.cl.uni-trier.de/',\n",
    "                            json={\n",
    "                                'model': MODEL,\n",
    "                                'system': SYSTEM_evidence,\n",
    "                                'prompt': f'Here is the set of claims to consider:\\n\"Claims\":\\n^^{claim_df.loc[:,'claims'][:5].to_list()}^^'\n",
    "                                            +'\\nHere is the set of social media comments to consider'+\n",
    "                                            f'\\n\"Comments\":\\n<{row['text']}>'\n",
    "                                            +'\\nCheck whether your answer consists of the specified JSON format and nothing else.'\n",
    "                                }).json()['response']                       \n",
    "                    ],\n",
    "                    columns=['reasons']\n",
    "                )\n",
    "                .assign(label=label)\n",
    "            )\n",
    "            \n",
    "        except json.JSONDecodeError:\n",
    "            print(\"invalid json response, skipping to next batch\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                             reasons  \\\n",
      "0   {\\n\"claims\": [\\n{\\n\"claim\": \"Ukraine's energy...   \n",
      "1   {\\n\"claims\": [\\n{\"claim\": \"Ukraine's energy s...   \n",
      "2   {\\n\"claims\": [\\n{\"claim\": \"Ukraine's energy s...   \n",
      "3   {\\n\"claims\": [\\n{\"claim\": \"Ukraine's energy s...   \n",
      "4   {\\n\"claims\": [\\n{\\n\"claim\": \"Ukraine's energy...   \n",
      "\n",
      "                                               label  \n",
      "0  https://www.wilsoncenter.org/blog-post/state-u...  \n",
      "1  https://www.wilsoncenter.org/blog-post/state-u...  \n",
      "2  https://www.wilsoncenter.org/blog-post/state-u...  \n",
      "3  https://www.wilsoncenter.org/blog-post/state-u...  \n",
      "4  https://www.wilsoncenter.org/blog-post/state-u...  \n"
     ]
    }
   ],
   "source": [
    "reason_df = pd.concat(chunked_result, ignore_index=True)\n",
    "print(reason_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "reason_df.to_csv(f'reasons.by.{GROUPER}.{LANGUAGE}_test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'id': '66530778-5585-11ef-8a8f-0d8e6fdd3d4b',\n",
       " 'timestamp': '2024-08-08T12:54:50.750527',\n",
       " 'model': 'llama3.1:70b-instruct-q6_K',\n",
       " 'prompt': [{'role': 'system',\n",
       "   'content': '\\n        Instruction:\\n\\n        You are a text annotation assitant. Analyze a collection of social media comments enclosed in chevrons <..>. Identify and list the claims within these comments. Claims can be related to events, issues, opinions or concerns in relation to the specified topic.\\n        Claims are defined as the main assertion or conclusion of an argument.\\n        You summarize each claim into a short simple sentence.\\n\\n        Response format:\\n\\n        You provide only the list of claims, separated by commas, without any additional text or explanations. If no claims can be identified, return an empty list [].\\n\\n        Response format template:\\n        \\n        [\"claim 1\", \"claim 2\", \"claim 3\"]\\n\\t'},\n",
       "  {'role': 'user',\n",
       "   'content': 'The following set of social media posts are about ukraine. Check whether your answer only consists of a list of claims. \\n\"Posts\":\\n<[\"Ukraine\\'s energy sector has undergone a remarkable transformation over the past decade, despite facing Russian threats. By adopting European-style legislation, strengthening energy security, and pursuing financial sustainability, Ukraine has made significant strides towards achieving transparent and secure energy markets.\", \"Ukraine\\'s energy sector has undergone significant changes over the past decade, despite ongoing conflicts. Efforts to align with European legislation and strengthen energy security have paid off, with notable achievements such as transparent markets and reduced state involvement. However, Russia\\'s threats persist, underscoring the need for continued resilience and adaptation.\", \"Ukraine\\'s energy sector has come a long way since the start of the war. Despite Russia\\'s continued threats, Ukraine has made significant strides in strengthening its energy security and achieving transparent energy markets. It\\'s time for the world to recognize Ukraine\\'s progress and support its efforts towards a more sustainable future.\", \"@UkraineEnergySector #Ukraine #EnergySector #RussiaIt\\'s been 10 years since the war started, and it\\'s high time to assess the impact on Ukraine\\'s energy sector. While Russia\\'s threats haven\\'t disappeared, Ukraine has made significant strides in strengthening its energy security & achieving transparency in the sector. 💡We\\'ve come a long way since 2014, adopting European-aligned laws & eliminating subsidies for households. The state\\'s role in the energy sector has been diminished, paving the way for private investment and competition. 💰Despite Russian aggression, Ukraine has shown resilience in its pursuit of energy independence. We must continue to support these efforts & ensure that our energy sector remains robust and secure. 💪\", \"Despite a decade of war, Ukraine\\'s energy sector shows resilience with legislative reforms aligning it with EU rules, moving towards transparent markets, & phasing out subsidies, while Russia continues threats through new tactics #UkraineEnergy 🇺🇦\"]>'}],\n",
       " 'response': '[\"Ukraine\\'s energy sector has undergone a remarkable transformation over the past decade\", \"Ukraine\\'s energy sector has made significant strides towards achieving transparent and secure energy markets\", \"Russia\\'s threats to Ukraine persist\", \"Ukraine has strengthened its energy security despite Russia\\'s continued threats\", \"The world should recognize Ukraine\\'s progress and support its efforts towards a more sustainable future\", \"Ukraine has eliminated subsidies for households in the energy sector\", \"Private investment and competition have increased in Ukraine\\'s energy sector due to diminished state involvement\", \"Ukraine must continue to pursue energy independence despite Russian aggression\"]'}"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#try chat functionality of api\n",
    "\n",
    "#test generalized claim mining\n",
    "requests.post('https://inf.cl.uni-trier.de/chat/',\n",
    "                json={\n",
    "                \"messages\": [\n",
    "                        {\"content\": SYSTEM_claim,\n",
    "                        \"role\": \"system\"\n",
    "                        },\n",
    "                        {\n",
    "                        \"content\": f'The following set of social media posts are about '\n",
    "                                + TOPIC + \n",
    "                                f'. Check whether your answer only consists of a list of claims. \\n\"Posts\":\\n<{selected_rows.loc[:,'text'][:5].to_list()}>',\n",
    "                        \"role\": \"user\"\n",
    "                        },\n",
    "                ],\n",
    "                \"model\": MODEL\n",
    "                }).json()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'id': 'ae0fa78c-5586-11ef-8a8f-0d8e6fdd3d4b',\n",
       " 'timestamp': '2024-08-08T13:04:00.601617',\n",
       " 'model': 'llama3.1:70b-instruct-q6_K',\n",
       " 'prompt': [{'role': 'user',\n",
       "   'content': 'Can you give me the five key claims made in these posts?'}],\n",
       " 'response': \"There are no posts to analyze. This conversation just started, and I haven't received any text to summarize. Please go ahead and provide the text, and I'll be happy to help you extract the key claims!\"}"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#test asking about previous reply\n",
    "requests.post('https://inf.cl.uni-trier.de/chat/',\n",
    "                json={\n",
    "                \"messages\": [\n",
    "                       {\n",
    "                        \"content\": 'Can you give me the five key claims made in these posts?',\n",
    "                        \"role\": \"user\"\n",
    "                        },\n",
    "                ],\n",
    "                \"model\": MODEL\n",
    "                }).json()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "#try asking the followup question in on go (so without knowing the response to the claim mining...)\n",
    "#first define the new task\n",
    "SYSTEM_chat: str = \\\n",
    "\"\"\"\n",
    "Instruction:\n",
    "\n",
    "You are a text annotation assitant. You read a collection of social media comments enclosed by chevrons <..>. You find the subtopics discussed within these comments, and the claims and premises made in regard to each of them. \n",
    "Use the following defenitions:\n",
    "1. Claim: The main assertion or conclusion of an argument\n",
    "2. Premise: Supporting evidence or reasons for a claim\n",
    "\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "USER_claim_chat: str = \\\n",
    "    \"\"\"Please identify and list the claims within these comments. Claims can be related to events, issues, opinions or concerns in relation to the specified topic.\n",
    "        Claims are defined as the main assertion or conclusion of an argument.\n",
    "        You summarize each claim into a short simple sentence.\\nResponse format:\n",
    "        You provide only the list of claims, separated by commas, without any additional text or explanations. If no claims can be identified, return an empty list [].\n",
    "        Response format template:\n",
    "        [\"claim 1\", \"claim 2\", \"claim 3\"]\n",
    "        The following set of social media posts are about\n",
    "    \"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "USER_premis_chat: str = \\\n",
    "    \"\"\"Now identify and list the reasons or evidence (if any) given within these comments for each of the claims. \n",
    "        A reason is defined as either:\n",
    "        1. A justification or argument that supports a particular claim.\n",
    "        2. An explanation that provides logical or empirical evidence to back up a claim.\n",
    "        You summarize each reason into a short simple sentence.\n",
    "        Response format:\n",
    "        You follow the JSON template below. If no claims or reasons can be identified, return an empty list [] in the respective JSON field.\n",
    "        Response format template:\n",
    "        {\n",
    "         \"claims\": [\n",
    "                    {\n",
    "                    \"claim\": \"Claim 1\",\n",
    "                    \"reasons\": [\n",
    "                                \"Reason 1\",\n",
    "                                \"Reason 2\",\n",
    "                                \"Reason 3\"\n",
    "                                ]\n",
    "                    }\n",
    "    \"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'[\"Ukraine\\'s energy sector has undergone a remarkable transformation over the past decade, despite facing Russian threats.\", \"Ukraine\\'s energy sector has made significant strides towards achieving transparent and secure energy markets.\", \"Ukraine has strengthened its energy security.\", \"Efforts to align with European legislation have paid off.\", \"Russia\\'s threats persist.\", \"Ukraine\\'s progress in strengthening its energy security should be recognized by the world.\", \"The state\\'s role in the energy sector has been diminished, paving the way for private investment and competition.\", \"Ukraine has shown resilience in its pursuit of energy independence.\", \"Despite Russian aggression, Ukraine has made significant strides in strengthening its energy security and achieving transparency in the sector.\", \"Ukraine\\'s energy sector shows resilience with legislative reforms aligning it with EU rules.\"]\\n\\n{\\n  \"claims\": [\\n    {\\n      \"claim\": \"Ukraine\\'s energy sector has undergone a remarkable transformation over the past decade, despite facing Russian threats.\",\\n      \"reasons\": [\\n        \"By adopting European-style legislation, strengthening energy security, and pursuing financial sustainability.\"\\n      ]\\n    },\\n    {\\n      \"claim\": \"Ukraine\\'s energy sector has made significant strides towards achieving transparent and secure energy markets.\",\\n      \"reasons\": [\\n        \"Efforts to align with European legislation have paid off, with notable achievements such as reduced state involvement.\"\\n      ]\\n    },\\n    {\\n      \"claim\": \"Ukraine has strengthened its energy security.\",\\n      \"reasons\": [\\n        \"Adopting European-aligned laws and eliminating subsidies for households.\"\\n      ]\\n    },\\n    {\\n      \"claim\": \"Efforts to align with European legislation have paid off.\",\\n      \"reasons\": [\\n        \"Notable achievements such as transparent markets and reduced state involvement.\"\\n      ]\\n    },\\n    {\\n      \"claim\": \"Russia\\'s threats persist.\",\\n      \"reasons\": []\\n    },\\n    {\\n      \"claim\": \"Ukraine\\'s progress in strengthening its energy security should be recognized by the world.\",\\n      \"reasons\": [\\n        \"Despite Russia\\'s continued threats, Ukraine has made significant strides in strengthening its energy security and achieving transparent energy markets.\"\\n      ]\\n    },\\n    {\\n      \"claim\": \"The state\\'s role in the energy sector has been diminished, paving the way for private investment and competition.\",\\n      \"reasons\": [\\n        \"Adopting European-aligned laws and eliminating subsidies for households.\"\\n      ]\\n    },\\n    {\\n      \"claim\": \"Ukraine has shown resilience in its pursuit of energy independence.\",\\n      \"reasons\": [\\n        \"Despite Russian aggression, Ukraine has made significant strides in strengthening its energy security and achieving transparency in the sector.\"\\n      ]\\n    },\\n    {\\n      \"claim\": \"Despite Russian aggression, Ukraine has made significant strides in strengthening its energy security and achieving transparency in the sector.\",\\n      \"reasons\": []\\n    },\\n    {\\n      \"claim\": \"Ukraine\\'s energy sector shows resilience with legislative reforms aligning it with EU rules.\",\\n      \"reasons\": [\\n        \"Moving towards transparent markets, phasing out subsidies.\"\\n      ]\\n    }\\n  ]\\n}'"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#test first asking for claims, then for premises:\n",
    "#test generalized claim mining\n",
    "requests.post('https://inf.cl.uni-trier.de/chat/',\n",
    "                json={\n",
    "                \"messages\": [\n",
    "                        {\"content\": SYSTEM_chat,\n",
    "                        \"role\": \"system\"\n",
    "                        },\n",
    "                        {\n",
    "                        \"content\": USER_claim_chat + TOPIC + \n",
    "                                f'. Check whether your answer only consists of a list of claims. \\n\"Posts\":\\n<{selected_rows.loc[:,'text'][:5].to_list()}>',\n",
    "                        \"role\": \"user\",\n",
    "                        },\n",
    "                        {\n",
    "                        \"content\": USER_premis_chat,\n",
    "                        \"role\": \"user\",\n",
    "                        }\n",
    "                        ],\n",
    "                \"model\": MODEL\n",
    "                }).json()['response']"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "DefiningDebateQuality",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}