gpt_benchmarking_using_biomixQA.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "3ccb4a25-2727-4a0f-8d62-7218e697c4c5",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/karthiksoman/anaconda3/lib/python3.10/site-packages/pandas/core/arrays/masked.py:60: UserWarning: Pandas requires version '1.3.6' or newer of 'bottleneck' (version '1.3.5' currently installed).\n",
      "  from pandas.core import (\n"
     ]
    }
   ],
   "source": [
    "from datasets import load_dataset\n",
    "from openai import OpenAI\n",
    "from dotenv import load_dotenv\n",
    "from joblib import Memory\n",
    "import os\n",
    "import json\n",
    "import pandas as pd\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e5bbddf1-8b62-443e-8b00-a6b85c779c09",
   "metadata": {},
   "source": [
    "## Load datasets from Hugging Face"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "62eff5a1-61ff-4d0f-88a9-82f3dd643dcd",
   "metadata": {},
   "outputs": [],
   "source": [
    "mcq_data = load_dataset(\"kg-rag/BiomixQA\", \"mcq\")\n",
    "\n",
    "tf_data = load_dataset(\"kg-rag/BiomixQA\", \"true_false\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c2c7454c-f37f-4b46-aad1-075c5903c78b",
   "metadata": {},
   "source": [
    "## MCQ data first sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "fb4bb5ec-de13-4478-a80a-6242063dab19",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'text': 'Out of the given list, which Gene is associated with head and neck cancer and uveal melanoma. Given list is:  ABO, CACNA2D1,  PSCA, TERT,  SULT1B1', 'option_A': 'ABO', 'option_B': 'CACNA2D1', 'option_C': 'PSCA', 'option_D': 'TERT', 'option_E': 'SULT1B1', 'correct_answer': 'CACNA2D1'}\n"
     ]
    }
   ],
   "source": [
    "print(mcq_data[\"train\"][0])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f4a1ac99-bd02-4437-ab17-f4a5ab5359cc",
   "metadata": {},
   "source": [
    "## True/False data first sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "60955b3d-26b7-45f1-9dff-5281191ca9a6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'text': 'enhanced S-cone syndrome is not a vitreoretinal degeneration', 'label': False}\n"
     ]
    }
   ],
   "source": [
    "print(tf_data[\"train\"][0])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3dbed3c9-039b-4d04-9618-ae9e5a20e71a",
   "metadata": {},
   "source": [
    "## Configure OpenAI client"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "7ef5ad66-cde9-4226-8b73-842501e5579b",
   "metadata": {},
   "outputs": [],
   "source": [
    "load_dotenv(os.path.join(os.path.expanduser('~'), '.gpt_biomixQA.env'))\n",
    "\n",
    "client = OpenAI(api_key = os.environ.get('API_KEY'))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "102099c8-6c6c-4045-a715-06fc5186523f",
   "metadata": {},
   "source": [
    "## Design System prompts for MCQ and True/False dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "4dac00f5-a07f-44d5-a73e-cbd3056a0c6a",
   "metadata": {},
   "outputs": [],
   "source": [
    "MCQ_QUESTION_SYSTEM_PROMPT = '''\n",
    "    You are an expert biomedical researcher. \n",
    "    Please provide your answer in the following JSON format for the Question asked:\n",
    "    {\"answer\": <correct answer>}\n",
    "'''\n",
    "\n",
    "TRUE_FALSE_QUESTION_SYSTEM_PROMPT = '''\n",
    "    You are an expert biomedical researcher. \n",
    "    Please provide your answer in the following JSON format for the Question asked:\n",
    "    {\"answer\": \"True\"}\n",
    "    OR\n",
    "    {\"answer\": \"False\"}\n",
    "'''"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5078ce07-1ac1-4cdc-a4d4-7f0f32d11a8b",
   "metadata": {},
   "source": [
    "## Selecting a GPT model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "711874e5-700b-4349-8cd4-660469d9b884",
   "metadata": {},
   "outputs": [],
   "source": [
    "CHAT_MODEL = \"gpt-4o\"\n",
    "\n",
    "TEMPERATURE = 0.3\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b32b2b95-7f30-4963-bbe5-6abdd6f75ee1",
   "metadata": {},
   "source": [
    "## Setting a cache memory for GPT calls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "02bdccb5-9836-4a76-83f2-d414e49535d8",
   "metadata": {},
   "outputs": [],
   "source": [
    "memory = Memory(\"cachegpt\", verbose=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ad2d363b-fb65-4781-8b58-4943af6bd424",
   "metadata": {},
   "source": [
    "## Custom function to call GPT model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "6a2d659a-57a2-41c5-aead-6d65d3af5eb1",
   "metadata": {},
   "outputs": [],
   "source": [
    "@memory.cache\n",
    "def call_GPT(instruction, system_prompt, chat_model_id, temperature):\n",
    "    response = client.chat.completions.create(        \n",
    "        temperature=temperature,\n",
    "        model=chat_model_id,\n",
    "        messages=[\n",
    "            {\"role\": \"system\", \"content\": system_prompt},\n",
    "            {\"role\": \"user\", \"content\": instruction}\n",
    "        ]\n",
    "    )\n",
    "    if response.choices:\n",
    "        return response.choices[0].message.content\n",
    "    else:\n",
    "        return 'Unexpected response'\n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d807ef15-e313-4b74-97b6-1c1ac234278f",
   "metadata": {},
   "source": [
    "## Evaluating GPT model on Biomix MCQ data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "7705cb33-64b9-455b-bcac-b5340004e690",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performance accuracy of gpt-4o on Biomix MCQ data is 68.3%\n"
     ]
    }
   ],
   "source": [
    "\n",
    "mcq_predictions = []\n",
    "for item in mcq_data[\"train\"]:\n",
    "    resp = call_GPT(item[\"text\"], MCQ_QUESTION_SYSTEM_PROMPT, CHAT_MODEL, TEMPERATURE) \n",
    "    try:\n",
    "        resp = json.loads(resp)\n",
    "        mcq_predictions.append((item[\"text\"], item[\"correct_answer\"], resp[\"answer\"]))\n",
    "    except json.JSONDecodeError as e:\n",
    "        continue\n",
    "\n",
    "mcq_prediction_df = pd.DataFrame(mcq_predictions, columns=[\"text\", \"correct_answer\", \"prediction\"])\n",
    "mcq_prediction_df_correct = mcq_prediction_df[mcq_prediction_df[\"correct_answer\"] == mcq_prediction_df[\"prediction\"]]\n",
    "mcq_accuracy = 100*mcq_prediction_df_correct.shape[0]/len(mcq_data[\"train\"])\n",
    "print(f\"Performance accuracy of {CHAT_MODEL} on Biomix MCQ data is {round(mcq_accuracy, 2)}%\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "20b880f8-2d7a-483c-8293-38f8e7bef691",
   "metadata": {},
   "source": [
    "## Evaluating GPT model on Biomix True/False data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "f2f57c7e-4eb9-4634-ac79-803d37ad110b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performance accuracy of gpt-4o on Biomix True/False data is 89.39%\n"
     ]
    }
   ],
   "source": [
    "\n",
    "tf_predictions = []\n",
    "for item in tf_data[\"train\"]:\n",
    "    resp = call_GPT(item[\"text\"], TRUE_FALSE_QUESTION_SYSTEM_PROMPT, CHAT_MODEL, TEMPERATURE) \n",
    "    try:\n",
    "        resp = json.loads(resp)\n",
    "        tf_predictions.append((item[\"text\"], item[\"label\"], resp[\"answer\"]))\n",
    "    except json.JSONDecodeError as e:\n",
    "        continue\n",
    "\n",
    "tf_predictions_df = pd.DataFrame(tf_predictions, columns=[\"text\", \"correct_answer\", \"prediction\"])\n",
    "tf_predictions_df.correct_answer = tf_predictions_df.correct_answer.astype(str)\n",
    "tf_predictions_df.prediction = tf_predictions_df.prediction.astype(str)\n",
    "tf_predictions_df_correct = tf_predictions_df[tf_predictions_df[\"correct_answer\"] == tf_predictions_df[\"prediction\"]]\n",
    "tf_accuracy = 100*tf_predictions_df_correct.shape[0]/len(tf_data[\"train\"])\n",
    "print(f\"Performance accuracy of {CHAT_MODEL} on Biomix True/False data is {round(tf_accuracy, 2)}%\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "135e81c3-58df-45ff-84df-708a91a182af",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}