Analysis_CancerQuery.ipynb

{
 "cells": [
  {
   "cell_type": "code",
<<<<<<< HEAD
   "execution_count": 2,
=======
   "execution_count": 4,
>>>>>>> da1efa8628124ffed82b84aafc58e2a7734db5bb
   "id": "31ae36bd",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
<<<<<<< HEAD
   "execution_count": 3,
=======
   "execution_count": 2,
>>>>>>> da1efa8628124ffed82b84aafc58e2a7734db5bb
   "id": "82c11f02",
   "metadata": {},
   "outputs": [],
   "source": [
    "#cancer_df = pd.read_csv('/data/team2/top10data/cancer.csv')\n",
    "#cancer_df['PMID'] = cancer_df['PMID'].apply(lambda x: x.split(','))\n",
    "#exploded_df = cancer_df.explode('PMID')\n",
    "#exploded_df = exploded_df.drop_duplicates('PMID')\n",
    "\n",
    "'''\n",
    "Files\n",
    "* API_results_top_bottom_10.csv - this contains all PMIDs returned for a given query\n",
    "* top10data/<query term>.csv - this contains data from users for a given query\n",
    "* pmid_fulltext_status.csv - this contains PMIDs and whether they have full text available\n",
    "* top_10_query_PMIDs_v2.csv - this contains metadata for PMIDs in returned by the top 10 queries\n",
    "\n",
    "Each of these\n",
    "'''\n",
    "\n",
    "processed_query = 'cancer'\n",
    "\n",
    "api_res_df = pd.read_csv('/data/team2/API_results_top_bottom_10.csv').rename(columns={'Unnamed: 0':'processed_query'})\n",
    "pq_pmids = api_res_df[api_res_df['processed_query'] == processed_query]\n",
    "\n",
    "# currently don't have metadata for all PMIDs in the API results dataframe\n",
    "top_10_query_PMIDs = pd.read_csv('/data/team2/top_10_query_PMIDs_v2.csv').drop('Unnamed: 0', axis=1)\n",
    "top_10_query_PMIDs['PMID'] = top_10_query_PMIDs['pmid'].astype(str)\n",
    "\n",
    "# adding hasAbstract, hasStructuredAbstract, pubtype\n",
    "merged_df = exploded_df.merge(top_10_query_PMIDs, how='left', on = 'PMID')\n",
    "\n",
    "# adding one-hot encoded publication types\n",
    "merged_df1 = merged_df.join(pd.get_dummies(merged_df.pubtype, prefix='pubtype'))\n",
    "\n",
    "# adding full-text availability\n",
    "fulltext = pd.read_csv('/data/team2/pmid_fulltext_status.csv').rename(columns={'Unnamed: 0':'PMID'})\n",
    "fulltext['PMID'] = fulltext['PMID'].astype(str)\n",
    "merged_df2 = merged_df1.merge(fulltext, on='PMID')\n",
    "fulltext_dummy = pd.get_dummies(merged_df2.fulltext_status)\n",
    "merged_df2['fulltext_status'] = fulltext_dummy['fulltext_available'] # 1 if it has fulltext, 0 if it doesn't"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "687c55b7",
   "metadata": {},
   "outputs": [],
   "source": [
    "fulltext = pd.read_csv('/data/team2/pmid_fulltext_status.csv').rename(columns={'Unnamed: 0':'PMID'})\n",
    "fulltext['PMID'] = fulltext['PMID'].astype(str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "454d1e3c",
   "metadata": {},
   "outputs": [],
   "source": [
    "merged_df2 = merged_df1.merge(fulltext, on='PMID')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "700918f6",
   "metadata": {},
   "outputs": [],
   "source": [
    "fulltext_dummy = pd.get_dummies(merged_df2.fulltext_status)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "490a278d",
   "metadata": {},
   "outputs": [],
   "source": [
    "merged_df2['fulltext_status'] = fulltext_dummy['fulltext_available'] # 1 if it has fulltext, 0 if it doesn't"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "63bbf7e6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(573, 294)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "merged_df2.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "d88e11ce",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>search_id</th>\n",
       "      <th>query_term</th>\n",
       "      <th>filters</th>\n",
       "      <th>sort_algorithm</th>\n",
       "      <th>result_count</th>\n",
       "      <th>page_num</th>\n",
       "      <th>PMID</th>\n",
       "      <th>click_data</th>\n",
       "      <th>date</th>\n",
       "      <th>user_id</th>\n",
       "      <th>...</th>\n",
       "      <th>pubtype_['Journal Article']</th>\n",
       "      <th>pubtype_['Letter', 'Comment']</th>\n",
       "      <th>pubtype_['Letter', 'Randomized Controlled Trial']</th>\n",
       "      <th>pubtype_['Letter']</th>\n",
       "      <th>pubtype_['Meta-Analysis', \"Research Support, Non-U.S. Gov't\", 'Systematic Review']</th>\n",
       "      <th>pubtype_['News', 'Comment']</th>\n",
       "      <th>pubtype_['News']</th>\n",
       "      <th>pubtype_['Published Erratum']</th>\n",
       "      <th>pubtype_['Retraction of Publication']</th>\n",
       "      <th>fulltext_status</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4ibte5dnudz3x_oeqibxuw:6dd943ee022371df5a9d5f0...</td>\n",
       "      <td>cancer</td>\n",
       "      <td>None</td>\n",
       "      <td>relevance</td>\n",
       "      <td>4569599</td>\n",
       "      <td>1</td>\n",
       "      <td>31761807</td>\n",
       "      <td>NoClicks</td>\n",
       "      <td>2022-03-16</td>\n",
       "      <td>Z392386</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4ibte5dnudz3x_oeqibxuw:6dd943ee022371df5a9d5f0...</td>\n",
       "      <td>cancer</td>\n",
       "      <td>None</td>\n",
       "      <td>relevance</td>\n",
       "      <td>4569599</td>\n",
       "      <td>1</td>\n",
       "      <td>28244479</td>\n",
       "      <td>NoClicks</td>\n",
       "      <td>2022-03-16</td>\n",
       "      <td>Z392386</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4ibte5dnudz3x_oeqibxuw:6dd943ee022371df5a9d5f0...</td>\n",
       "      <td>cancer</td>\n",
       "      <td>None</td>\n",
       "      <td>relevance</td>\n",
       "      <td>4569599</td>\n",
       "      <td>1</td>\n",
       "      <td>27741350</td>\n",
       "      <td>NoClicks</td>\n",
       "      <td>2022-03-16</td>\n",
       "      <td>Z392386</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4ibte5dnudz3x_oeqibxuw:6dd943ee022371df5a9d5f0...</td>\n",
       "      <td>cancer</td>\n",
       "      <td>None</td>\n",
       "      <td>relevance</td>\n",
       "      <td>4569599</td>\n",
       "      <td>1</td>\n",
       "      <td>30713326</td>\n",
       "      <td>NoClicks</td>\n",
       "      <td>2022-03-16</td>\n",
       "      <td>Z392386</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4ibte5dnudz3x_oeqibxuw:6dd943ee022371df5a9d5f0...</td>\n",
       "      <td>cancer</td>\n",
       "      <td>None</td>\n",
       "      <td>relevance</td>\n",
       "      <td>4569599</td>\n",
       "      <td>1</td>\n",
       "      <td>29949179</td>\n",
       "      <td>NoClicks</td>\n",
       "      <td>2022-03-16</td>\n",
       "      <td>Z392386</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 294 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           search_id query_term filters  \\\n",
       "0  4ibte5dnudz3x_oeqibxuw:6dd943ee022371df5a9d5f0...     cancer    None   \n",
       "1  4ibte5dnudz3x_oeqibxuw:6dd943ee022371df5a9d5f0...     cancer    None   \n",
       "2  4ibte5dnudz3x_oeqibxuw:6dd943ee022371df5a9d5f0...     cancer    None   \n",
       "3  4ibte5dnudz3x_oeqibxuw:6dd943ee022371df5a9d5f0...     cancer    None   \n",
       "4  4ibte5dnudz3x_oeqibxuw:6dd943ee022371df5a9d5f0...     cancer    None   \n",
       "\n",
       "  sort_algorithm  result_count  page_num      PMID click_data        date  \\\n",
       "0      relevance       4569599         1  31761807   NoClicks  2022-03-16   \n",
       "1      relevance       4569599         1  28244479   NoClicks  2022-03-16   \n",
       "2      relevance       4569599         1  27741350   NoClicks  2022-03-16   \n",
       "3      relevance       4569599         1  30713326   NoClicks  2022-03-16   \n",
       "4      relevance       4569599         1  29949179   NoClicks  2022-03-16   \n",
       "\n",
       "   user_id  ...  pubtype_['Journal Article'] pubtype_['Letter', 'Comment']  \\\n",
       "0  Z392386  ...                            1                             0   \n",
       "1  Z392386  ...                            0                             0   \n",
       "2  Z392386  ...                            0                             0   \n",
       "3  Z392386  ...                            0                             0   \n",
       "4  Z392386  ...                            0                             0   \n",
       "\n",
       "   pubtype_['Letter', 'Randomized Controlled Trial'] pubtype_['Letter']  \\\n",
       "0                                                  0                  0   \n",
       "1                                                  0                  0   \n",
       "2                                                  0                  0   \n",
       "3                                                  0                  0   \n",
       "4                                                  0                  0   \n",
       "\n",
       "  pubtype_['Meta-Analysis', \"Research Support, Non-U.S. Gov't\", 'Systematic Review']  \\\n",
       "0                                                  0                                   \n",
       "1                                                  0                                   \n",
       "2                                                  0                                   \n",
       "3                                                  0                                   \n",
       "4                                                  0                                   \n",
       "\n",
       "  pubtype_['News', 'Comment'] pubtype_['News'] pubtype_['Published Erratum']  \\\n",
       "0                           0                0                             0   \n",
       "1                           0                0                             0   \n",
       "2                           0                0                             0   \n",
       "3                           0                0                             0   \n",
       "4                           0                0                             0   \n",
       "\n",
       "  pubtype_['Retraction of Publication'] fulltext_status  \n",
       "0                                     0               0  \n",
       "1                                     0               1  \n",
       "2                                     0               0  \n",
       "3                                     0               1  \n",
       "4                                     0               1  \n",
       "\n",
       "[5 rows x 294 columns]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "merged_df2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "0e9337dc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>search_id</th>\n",
       "      <th>query_term</th>\n",
       "      <th>filters</th>\n",
       "      <th>sort_algorithm</th>\n",
       "      <th>result_count</th>\n",
       "      <th>page_num</th>\n",
       "      <th>PMID</th>\n",
       "      <th>click_data</th>\n",
       "      <th>date</th>\n",
       "      <th>user_id</th>\n",
       "      <th>filters_active</th>\n",
       "      <th>processed_query</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4ibte5dnudz3x_oeqibxuw:6dd943ee022371df5a9d5f0...</td>\n",
       "      <td>cancer</td>\n",
       "      <td>None</td>\n",
       "      <td>relevance</td>\n",
       "      <td>4569599</td>\n",
       "      <td>1</td>\n",
       "      <td>[31761807, 28244479, 27741350, 30713326, 29949...</td>\n",
       "      <td>NoClicks</td>\n",
       "      <td>2022-03-16</td>\n",
       "      <td>Z392386</td>\n",
       "      <td>False</td>\n",
       "      <td>cancer</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5cr7r_f0i_6wyd0xw3ayza:93764b25ae99d315f410802...</td>\n",
       "      <td>cancer</td>\n",
       "      <td>None</td>\n",
       "      <td>date</td>\n",
       "      <td>4569599</td>\n",
       "      <td>1</td>\n",
       "      <td>[35289556, 35289548, 35289536, 35289535, 35289...</td>\n",
       "      <td>NoClicks</td>\n",
       "      <td>2022-03-16</td>\n",
       "      <td>Z401427</td>\n",
       "      <td>False</td>\n",
       "      <td>cancer</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>_hyofx-kvkxkzrumj0tvoa:a99dca5593185c498b63a5e...</td>\n",
       "      <td>cancer</td>\n",
       "      <td>None</td>\n",
       "      <td>relevance</td>\n",
       "      <td>4569599</td>\n",
       "      <td>1</td>\n",
       "      <td>[31761807, 28244479, 27741350, 30713326, 29949...</td>\n",
       "      <td>*result_click,28244479,2022-03-16T20:03:49,1,F...</td>\n",
       "      <td>2022-03-16</td>\n",
       "      <td>Z272979</td>\n",
       "      <td>False</td>\n",
       "      <td>cancer</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>_wbpom97rhxkmccuzyb26g:7463543d784aa59ca86359a...</td>\n",
       "      <td>cancer</td>\n",
       "      <td>None</td>\n",
       "      <td>relevance</td>\n",
       "      <td>4569599</td>\n",
       "      <td>1</td>\n",
       "      <td>[31761807, 28244479, 27741350, 30713326, 29949...</td>\n",
       "      <td>NoClicks</td>\n",
       "      <td>2022-03-16</td>\n",
       "      <td>Z93392</td>\n",
       "      <td>False</td>\n",
       "      <td>cancer</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>aajrft_asz6yj4tjzrqkdq:26985a88de1e140c5bce6ac...</td>\n",
       "      <td>cancer</td>\n",
       "      <td>datesearch.y_5</td>\n",
       "      <td>relevance</td>\n",
       "      <td>1064948</td>\n",
       "      <td>1</td>\n",
       "      <td>[29860986, 31761807, 30445141, 28154351, 30713...</td>\n",
       "      <td>*result_click,31761807,2022-03-16T20:44:35,1,F...</td>\n",
       "      <td>2022-03-16</td>\n",
       "      <td>Z292543</td>\n",
       "      <td>True</td>\n",
       "      <td>cancer</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           search_id query_term  \\\n",
       "0  4ibte5dnudz3x_oeqibxuw:6dd943ee022371df5a9d5f0...     cancer   \n",
       "1  5cr7r_f0i_6wyd0xw3ayza:93764b25ae99d315f410802...     cancer   \n",
       "2  _hyofx-kvkxkzrumj0tvoa:a99dca5593185c498b63a5e...     cancer   \n",
       "3  _wbpom97rhxkmccuzyb26g:7463543d784aa59ca86359a...     cancer   \n",
       "4  aajrft_asz6yj4tjzrqkdq:26985a88de1e140c5bce6ac...     cancer   \n",
       "\n",
       "          filters sort_algorithm  result_count  page_num  \\\n",
       "0            None      relevance       4569599         1   \n",
       "1            None           date       4569599         1   \n",
       "2            None      relevance       4569599         1   \n",
       "3            None      relevance       4569599         1   \n",
       "4  datesearch.y_5      relevance       1064948         1   \n",
       "\n",
       "                                                PMID  \\\n",
       "0  [31761807, 28244479, 27741350, 30713326, 29949...   \n",
       "1  [35289556, 35289548, 35289536, 35289535, 35289...   \n",
       "2  [31761807, 28244479, 27741350, 30713326, 29949...   \n",
       "3  [31761807, 28244479, 27741350, 30713326, 29949...   \n",
       "4  [29860986, 31761807, 30445141, 28154351, 30713...   \n",
       "\n",
       "                                          click_data        date  user_id  \\\n",
       "0                                           NoClicks  2022-03-16  Z392386   \n",
       "1                                           NoClicks  2022-03-16  Z401427   \n",
       "2  *result_click,28244479,2022-03-16T20:03:49,1,F...  2022-03-16  Z272979   \n",
       "3                                           NoClicks  2022-03-16   Z93392   \n",
       "4  *result_click,31761807,2022-03-16T20:44:35,1,F...  2022-03-16  Z292543   \n",
       "\n",
       "   filters_active processed_query  \n",
       "0           False          cancer  \n",
       "1           False          cancer  \n",
       "2           False          cancer  \n",
       "3           False          cancer  \n",
       "4            True          cancer  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cancer_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "e9d956c2",
   "metadata": {},
   "outputs": [],
   "source": [
    "api_res = pd.read_csv('/data/team2/API_results_top_bottom_10.csv').rename(columns={'Unnamed: 0':'processed_query'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "9fc6153d",
   "metadata": {},
   "outputs": [],
   "source": [
    "bc_list = api_res.iloc[0,1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "ea6faea1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "117227"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(bc_list)"
   ]
  },
  {
   "cell_type": "code",
<<<<<<< HEAD
   "execution_count": 30,
   "id": "31ee73ca",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(16819, 11)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top_10_query_PMIDs.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "6e7c6abc",
=======
   "execution_count": 3,
   "id": "64f33434",
>>>>>>> da1efa8628124ffed82b84aafc58e2a7734db5bb
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
<<<<<<< HEAD
       "      <th>pmid</th>\n",
       "      <th>journal</th>\n",
       "      <th>title</th>\n",
       "      <th>abstract</th>\n",
       "      <th>hasAbstract</th>\n",
       "      <th>hasStructuredAbstract</th>\n",
       "      <th>articleDate</th>\n",
       "      <th>journalPubDate</th>\n",
       "      <th>entrezDate</th>\n",
       "      <th>pubtype</th>\n",
       "      <th>PMID</th>\n",
=======
       "      <th>search_id</th>\n",
       "      <th>query_term</th>\n",
       "      <th>filters</th>\n",
       "      <th>sort_algorithm</th>\n",
       "      <th>result_count</th>\n",
       "      <th>page_num</th>\n",
       "      <th>PMID</th>\n",
       "      <th>click_data</th>\n",
       "      <th>date</th>\n",
       "      <th>user_id</th>\n",
       "      <th>...</th>\n",
       "      <th>pubtype_['Journal Article', 'Validation Study']</th>\n",
       "      <th>pubtype_['Journal Article']</th>\n",
       "      <th>pubtype_['Letter', 'Comment']</th>\n",
       "      <th>pubtype_['Letter', 'Randomized Controlled Trial']</th>\n",
       "      <th>pubtype_['Letter']</th>\n",
       "      <th>pubtype_['Meta-Analysis', \"Research Support, Non-U.S. Gov't\", 'Systematic Review']</th>\n",
       "      <th>pubtype_['News', 'Comment']</th>\n",
       "      <th>pubtype_['News']</th>\n",
       "      <th>pubtype_['Published Erratum']</th>\n",
       "      <th>pubtype_['Retraction of Publication']</th>\n",
>>>>>>> da1efa8628124ffed82b84aafc58e2a7734db5bb
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
<<<<<<< HEAD
       "      <td>35252802</td>\n",
       "      <td>Journal of biosafety and biosecurity</td>\n",
       "      <td>Recent developments and trends of automatic nu...</td>\n",
       "      <td>Nucleic acid detection, widely used in clinica...</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>2022-02-28</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2022-03-07</td>\n",
       "      <td>['Journal Article', 'Review']</td>\n",
       "      <td>35252802</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>35287656</td>\n",
       "      <td>BMC public health</td>\n",
       "      <td>Nutrition standards for the charitable food sy...</td>\n",
       "      <td>Food insecurity is associated with increased r...</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>2022-03-14</td>\n",
       "      <td>2022-03-14</td>\n",
       "      <td>2022-03-15</td>\n",
       "      <td>['Journal Article', \"Research Support, Non-U.S...</td>\n",
       "      <td>35287656</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>35232441</td>\n",
       "      <td>BMC medical education</td>\n",
       "      <td>Establishing CREATE: lessons learned in settin...</td>\n",
       "      <td>[{'@Label': 'BACKGROUND', '@NlmCategory': 'BAC...</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>2022-03-01</td>\n",
       "      <td>2022-03-01</td>\n",
       "      <td>2022-03-02</td>\n",
       "      <td>['Journal Article']</td>\n",
       "      <td>35232441</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>32282054</td>\n",
       "      <td>Turk patoloji dergisi</td>\n",
       "      <td>Pulmonary Acantholytic Squamous Cell Carcinoma...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2020-04-14</td>\n",
       "      <td>['Journal Article']</td>\n",
       "      <td>32282054</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>33629745</td>\n",
       "      <td>Journal of cellular physiology</td>\n",
       "      <td>TRIM11 suppresses ferritinophagy and gemcitabi...</td>\n",
       "      <td>Gemcitabine is first-line chemotherapy for pan...</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>2021-02-25</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2021-02-25</td>\n",
       "      <td>['Journal Article', \"Research Support, Non-U.S...</td>\n",
       "      <td>33629745</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       pmid                               journal  \\\n",
       "0  35252802  Journal of biosafety and biosecurity   \n",
       "1  35287656                     BMC public health   \n",
       "2  35232441                 BMC medical education   \n",
       "3  32282054                 Turk patoloji dergisi   \n",
       "4  33629745        Journal of cellular physiology   \n",
       "\n",
       "                                               title  \\\n",
       "0  Recent developments and trends of automatic nu...   \n",
       "1  Nutrition standards for the charitable food sy...   \n",
       "2  Establishing CREATE: lessons learned in settin...   \n",
       "3  Pulmonary Acantholytic Squamous Cell Carcinoma...   \n",
       "4  TRIM11 suppresses ferritinophagy and gemcitabi...   \n",
       "\n",
       "                                            abstract  hasAbstract  \\\n",
       "0  Nucleic acid detection, widely used in clinica...         True   \n",
       "1  Food insecurity is associated with increased r...         True   \n",
       "2  [{'@Label': 'BACKGROUND', '@NlmCategory': 'BAC...         True   \n",
       "3                                                NaN        False   \n",
       "4  Gemcitabine is first-line chemotherapy for pan...         True   \n",
       "\n",
       "   hasStructuredAbstract articleDate journalPubDate  entrezDate  \\\n",
       "0                  False  2022-02-28            NaN  2022-03-07   \n",
       "1                  False  2022-03-14     2022-03-14  2022-03-15   \n",
       "2                   True  2022-03-01     2022-03-01  2022-03-02   \n",
       "3                  False         NaN            NaN  2020-04-14   \n",
       "4                  False  2021-02-25            NaN  2021-02-25   \n",
       "\n",
       "                                             pubtype      PMID  \n",
       "0                      ['Journal Article', 'Review']  35252802  \n",
       "1  ['Journal Article', \"Research Support, Non-U.S...  35287656  \n",
       "2                                ['Journal Article']  35232441  \n",
       "3                                ['Journal Article']  32282054  \n",
       "4  ['Journal Article', \"Research Support, Non-U.S...  33629745  "
      ]
     },
     "execution_count": 33,
=======
       "      <td>4ibte5dnudz3x_oeqibxuw:6dd943ee022371df5a9d5f0...</td>\n",
       "      <td>cancer</td>\n",
       "      <td>None</td>\n",
       "      <td>relevance</td>\n",
       "      <td>4569599</td>\n",
       "      <td>1</td>\n",
       "      <td>31761807</td>\n",
       "      <td>NoClicks</td>\n",
       "      <td>2022-03-16</td>\n",
       "      <td>Z392386</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4ibte5dnudz3x_oeqibxuw:6dd943ee022371df5a9d5f0...</td>\n",
       "      <td>cancer</td>\n",
       "      <td>None</td>\n",
       "      <td>relevance</td>\n",
       "      <td>4569599</td>\n",
       "      <td>1</td>\n",
       "      <td>28244479</td>\n",
       "      <td>NoClicks</td>\n",
       "      <td>2022-03-16</td>\n",
       "      <td>Z392386</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4ibte5dnudz3x_oeqibxuw:6dd943ee022371df5a9d5f0...</td>\n",
       "      <td>cancer</td>\n",
       "      <td>None</td>\n",
       "      <td>relevance</td>\n",
       "      <td>4569599</td>\n",
       "      <td>1</td>\n",
       "      <td>27741350</td>\n",
       "      <td>NoClicks</td>\n",
       "      <td>2022-03-16</td>\n",
       "      <td>Z392386</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4ibte5dnudz3x_oeqibxuw:6dd943ee022371df5a9d5f0...</td>\n",
       "      <td>cancer</td>\n",
       "      <td>None</td>\n",
       "      <td>relevance</td>\n",
       "      <td>4569599</td>\n",
       "      <td>1</td>\n",
       "      <td>30713326</td>\n",
       "      <td>NoClicks</td>\n",
       "      <td>2022-03-16</td>\n",
       "      <td>Z392386</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4ibte5dnudz3x_oeqibxuw:6dd943ee022371df5a9d5f0...</td>\n",
       "      <td>cancer</td>\n",
       "      <td>None</td>\n",
       "      <td>relevance</td>\n",
       "      <td>4569599</td>\n",
       "      <td>1</td>\n",
       "      <td>29949179</td>\n",
       "      <td>NoClicks</td>\n",
       "      <td>2022-03-16</td>\n",
       "      <td>Z392386</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 293 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           search_id query_term filters  \\\n",
       "0  4ibte5dnudz3x_oeqibxuw:6dd943ee022371df5a9d5f0...     cancer    None   \n",
       "1  4ibte5dnudz3x_oeqibxuw:6dd943ee022371df5a9d5f0...     cancer    None   \n",
       "2  4ibte5dnudz3x_oeqibxuw:6dd943ee022371df5a9d5f0...     cancer    None   \n",
       "3  4ibte5dnudz3x_oeqibxuw:6dd943ee022371df5a9d5f0...     cancer    None   \n",
       "4  4ibte5dnudz3x_oeqibxuw:6dd943ee022371df5a9d5f0...     cancer    None   \n",
       "\n",
       "  sort_algorithm  result_count  page_num      PMID click_data        date  \\\n",
       "0      relevance       4569599         1  31761807   NoClicks  2022-03-16   \n",
       "1      relevance       4569599         1  28244479   NoClicks  2022-03-16   \n",
       "2      relevance       4569599         1  27741350   NoClicks  2022-03-16   \n",
       "3      relevance       4569599         1  30713326   NoClicks  2022-03-16   \n",
       "4      relevance       4569599         1  29949179   NoClicks  2022-03-16   \n",
       "\n",
       "   user_id  ...  pubtype_['Journal Article', 'Validation Study']  \\\n",
       "0  Z392386  ...                                                0   \n",
       "1  Z392386  ...                                                0   \n",
       "2  Z392386  ...                                                0   \n",
       "3  Z392386  ...                                                0   \n",
       "4  Z392386  ...                                                0   \n",
       "\n",
       "  pubtype_['Journal Article']  pubtype_['Letter', 'Comment']  \\\n",
       "0                           1                              0   \n",
       "1                           0                              0   \n",
       "2                           0                              0   \n",
       "3                           0                              0   \n",
       "4                           0                              0   \n",
       "\n",
       "  pubtype_['Letter', 'Randomized Controlled Trial'] pubtype_['Letter']  \\\n",
       "0                                                 0                  0   \n",
       "1                                                 0                  0   \n",
       "2                                                 0                  0   \n",
       "3                                                 0                  0   \n",
       "4                                                 0                  0   \n",
       "\n",
       "  pubtype_['Meta-Analysis', \"Research Support, Non-U.S. Gov't\", 'Systematic Review']  \\\n",
       "0                                                  0                                   \n",
       "1                                                  0                                   \n",
       "2                                                  0                                   \n",
       "3                                                  0                                   \n",
       "4                                                  0                                   \n",
       "\n",
       "  pubtype_['News', 'Comment'] pubtype_['News'] pubtype_['Published Erratum']  \\\n",
       "0                           0                0                             0   \n",
       "1                           0                0                             0   \n",
       "2                           0                0                             0   \n",
       "3                           0                0                             0   \n",
       "4                           0                0                             0   \n",
       "\n",
       "  pubtype_['Retraction of Publication']  \n",
       "0                                     0  \n",
       "1                                     0  \n",
       "2                                     0  \n",
       "3                                     0  \n",
       "4                                     0  \n",
       "\n",
       "[5 rows x 293 columns]"
      ]
     },
     "execution_count": 3,
>>>>>>> da1efa8628124ffed82b84aafc58e2a7734db5bb
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
<<<<<<< HEAD
    "top_10_query_PMIDs.head()"
=======
    "merged_df1.head()"
>>>>>>> da1efa8628124ffed82b84aafc58e2a7734db5bb
   ]
  },
  {
   "cell_type": "code",
<<<<<<< HEAD
   "execution_count": 31,
   "id": "278d7f45",
=======
   "execution_count": 7,
   "id": "80d746d4",
>>>>>>> da1efa8628124ffed82b84aafc58e2a7734db5bb
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
<<<<<<< HEAD
       "(439, 12)"
      ]
     },
     "execution_count": 31,
=======
       "2142"
      ]
     },
     "execution_count": 7,
>>>>>>> da1efa8628124ffed82b84aafc58e2a7734db5bb
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
<<<<<<< HEAD
    "cancer_df.shape"
=======
    "len(merged_df)"
>>>>>>> da1efa8628124ffed82b84aafc58e2a7734db5bb
   ]
  },
  {
   "cell_type": "code",
<<<<<<< HEAD
   "execution_count": 32,
   "id": "59d1495a",
=======
   "execution_count": 5,
   "id": "d19db3af",
>>>>>>> da1efa8628124ffed82b84aafc58e2a7734db5bb
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>search_id</th>\n",
       "      <th>query_term</th>\n",
       "      <th>filters</th>\n",
       "      <th>sort_algorithm</th>\n",
       "      <th>result_count</th>\n",
       "      <th>page_num</th>\n",
       "      <th>PMID</th>\n",
       "      <th>click_data</th>\n",
       "      <th>date</th>\n",
       "      <th>user_id</th>\n",
       "      <th>filters_active</th>\n",
       "      <th>processed_query</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4ibte5dnudz3x_oeqibxuw:6dd943ee022371df5a9d5f0...</td>\n",
       "      <td>cancer</td>\n",
       "      <td>None</td>\n",
       "      <td>relevance</td>\n",
       "      <td>4569599</td>\n",
       "      <td>1</td>\n",
<<<<<<< HEAD
       "      <td>[31761807, 28244479, 27741350, 30713326, 29949...</td>\n",
=======
       "      <td>31761807,28244479,27741350,30713326,29949179,2...</td>\n",
>>>>>>> da1efa8628124ffed82b84aafc58e2a7734db5bb
       "      <td>NoClicks</td>\n",
       "      <td>2022-03-16</td>\n",
       "      <td>Z392386</td>\n",
       "      <td>False</td>\n",
       "      <td>cancer</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5cr7r_f0i_6wyd0xw3ayza:93764b25ae99d315f410802...</td>\n",
       "      <td>cancer</td>\n",
       "      <td>None</td>\n",
       "      <td>date</td>\n",
       "      <td>4569599</td>\n",
       "      <td>1</td>\n",
<<<<<<< HEAD
       "      <td>[35289556, 35289548, 35289536, 35289535, 35289...</td>\n",
=======
       "      <td>35289556,35289548,35289536,35289535,35289519,3...</td>\n",
>>>>>>> da1efa8628124ffed82b84aafc58e2a7734db5bb
       "      <td>NoClicks</td>\n",
       "      <td>2022-03-16</td>\n",
       "      <td>Z401427</td>\n",
       "      <td>False</td>\n",
       "      <td>cancer</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>_hyofx-kvkxkzrumj0tvoa:a99dca5593185c498b63a5e...</td>\n",
       "      <td>cancer</td>\n",
       "      <td>None</td>\n",
       "      <td>relevance</td>\n",
       "      <td>4569599</td>\n",
       "      <td>1</td>\n",
<<<<<<< HEAD
       "      <td>[31761807, 28244479, 27741350, 30713326, 29949...</td>\n",
=======
       "      <td>31761807,28244479,27741350,30713326,29949179,2...</td>\n",
>>>>>>> da1efa8628124ffed82b84aafc58e2a7734db5bb
       "      <td>*result_click,28244479,2022-03-16T20:03:49,1,F...</td>\n",
       "      <td>2022-03-16</td>\n",
       "      <td>Z272979</td>\n",
       "      <td>False</td>\n",
       "      <td>cancer</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>_wbpom97rhxkmccuzyb26g:7463543d784aa59ca86359a...</td>\n",
       "      <td>cancer</td>\n",
       "      <td>None</td>\n",
       "      <td>relevance</td>\n",
       "      <td>4569599</td>\n",
       "      <td>1</td>\n",
<<<<<<< HEAD
       "      <td>[31761807, 28244479, 27741350, 30713326, 29949...</td>\n",
=======
       "      <td>31761807,28244479,27741350,30713326,29949179,2...</td>\n",
>>>>>>> da1efa8628124ffed82b84aafc58e2a7734db5bb
       "      <td>NoClicks</td>\n",
       "      <td>2022-03-16</td>\n",
       "      <td>Z93392</td>\n",
       "      <td>False</td>\n",
       "      <td>cancer</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>aajrft_asz6yj4tjzrqkdq:26985a88de1e140c5bce6ac...</td>\n",
       "      <td>cancer</td>\n",
       "      <td>datesearch.y_5</td>\n",
       "      <td>relevance</td>\n",
       "      <td>1064948</td>\n",
       "      <td>1</td>\n",
<<<<<<< HEAD
       "      <td>[29860986, 31761807, 30445141, 28154351, 30713...</td>\n",
=======
       "      <td>29860986,31761807,30445141,28154351,30713326,2...</td>\n",
>>>>>>> da1efa8628124ffed82b84aafc58e2a7734db5bb
       "      <td>*result_click,31761807,2022-03-16T20:44:35,1,F...</td>\n",
       "      <td>2022-03-16</td>\n",
       "      <td>Z292543</td>\n",
       "      <td>True</td>\n",
       "      <td>cancer</td>\n",
       "    </tr>\n",
<<<<<<< HEAD
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           search_id query_term  \\\n",
       "0  4ibte5dnudz3x_oeqibxuw:6dd943ee022371df5a9d5f0...     cancer   \n",
       "1  5cr7r_f0i_6wyd0xw3ayza:93764b25ae99d315f410802...     cancer   \n",
       "2  _hyofx-kvkxkzrumj0tvoa:a99dca5593185c498b63a5e...     cancer   \n",
       "3  _wbpom97rhxkmccuzyb26g:7463543d784aa59ca86359a...     cancer   \n",
       "4  aajrft_asz6yj4tjzrqkdq:26985a88de1e140c5bce6ac...     cancer   \n",
       "\n",
       "          filters sort_algorithm  result_count  page_num  \\\n",
       "0            None      relevance       4569599         1   \n",
       "1            None           date       4569599         1   \n",
       "2            None      relevance       4569599         1   \n",
       "3            None      relevance       4569599         1   \n",
       "4  datesearch.y_5      relevance       1064948         1   \n",
       "\n",
       "                                                PMID  \\\n",
       "0  [31761807, 28244479, 27741350, 30713326, 29949...   \n",
       "1  [35289556, 35289548, 35289536, 35289535, 35289...   \n",
       "2  [31761807, 28244479, 27741350, 30713326, 29949...   \n",
       "3  [31761807, 28244479, 27741350, 30713326, 29949...   \n",
       "4  [29860986, 31761807, 30445141, 28154351, 30713...   \n",
       "\n",
       "                                          click_data        date  user_id  \\\n",
       "0                                           NoClicks  2022-03-16  Z392386   \n",
       "1                                           NoClicks  2022-03-16  Z401427   \n",
       "2  *result_click,28244479,2022-03-16T20:03:49,1,F...  2022-03-16  Z272979   \n",
       "3                                           NoClicks  2022-03-16   Z93392   \n",
       "4  *result_click,31761807,2022-03-16T20:44:35,1,F...  2022-03-16  Z292543   \n",
       "\n",
       "   filters_active processed_query  \n",
       "0           False          cancer  \n",
       "1           False          cancer  \n",
       "2           False          cancer  \n",
       "3           False          cancer  \n",
       "4            True          cancer  "
      ]
     },
     "execution_count": 32,
=======
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>434</th>\n",
       "      <td>zh8ka-z38c5-lt2u_es5pa:a99dca5593185c498b63a5e...</td>\n",
       "      <td>cancer</td>\n",
       "      <td>None</td>\n",
       "      <td>relevance</td>\n",
       "      <td>4565187</td>\n",
       "      <td>1</td>\n",
       "      <td>31761807,28244479,27741350,30713326,29949179,2...</td>\n",
       "      <td>NoClicks</td>\n",
       "      <td>2022-03-09</td>\n",
       "      <td>Z391060</td>\n",
       "      <td>False</td>\n",
       "      <td>cancer</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>435</th>\n",
       "      <td>zixesdd3x1a8dq8qf40noa:bce3586a1d3a3cd33201d84...</td>\n",
       "      <td>cancer</td>\n",
       "      <td>pubt.review,lang.portuguese</td>\n",
       "      <td>relevance</td>\n",
       "      <td>801</td>\n",
       "      <td>1</td>\n",
       "      <td>28614430,29160494,32354458,29538545,32667426,2...</td>\n",
       "      <td>*result_click,29538545,2022-03-09T09:04:47,1,F...</td>\n",
       "      <td>2022-03-09</td>\n",
       "      <td>Z141330</td>\n",
       "      <td>True</td>\n",
       "      <td>cancer</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>436</th>\n",
       "      <td>zvydnkesoc5_lxcvysq50g:c2ba8420ea8e9b67e985e4d...</td>\n",
       "      <td>cancer</td>\n",
       "      <td>None</td>\n",
       "      <td>relevance</td>\n",
       "      <td>4565187</td>\n",
       "      <td>1</td>\n",
       "      <td>31761807,28244479,27741350,30713326,29949179,2...</td>\n",
       "      <td>NoClicks</td>\n",
       "      <td>2022-03-09</td>\n",
       "      <td>Z201014</td>\n",
       "      <td>False</td>\n",
       "      <td>cancer</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>437</th>\n",
       "      <td>zx-dmpxxkv1fk8ustboiia:6a1cd9549015db5492d2984...</td>\n",
       "      <td>cancer</td>\n",
       "      <td>simsearch2.ffrft</td>\n",
       "      <td>relevance</td>\n",
       "      <td>1431624</td>\n",
       "      <td>1</td>\n",
       "      <td>18538731,29860986,28244479,28154351,30713326,2...</td>\n",
       "      <td>NoClicks</td>\n",
       "      <td>2022-03-09</td>\n",
       "      <td>Z381937</td>\n",
       "      <td>True</td>\n",
       "      <td>cancer</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>438</th>\n",
       "      <td>zzd1qvexmjxhr6bzsoar2w:a99dca5593185c498b63a5e...</td>\n",
       "      <td>cancer</td>\n",
       "      <td>None</td>\n",
       "      <td>relevance</td>\n",
       "      <td>4565187</td>\n",
       "      <td>1</td>\n",
       "      <td>31761807,28244479,27741350,30713326,29949179,2...</td>\n",
       "      <td>NoClicks</td>\n",
       "      <td>2022-03-09</td>\n",
       "      <td>Z237552</td>\n",
       "      <td>False</td>\n",
       "      <td>cancer</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>439 rows × 12 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             search_id query_term  \\\n",
       "0    4ibte5dnudz3x_oeqibxuw:6dd943ee022371df5a9d5f0...     cancer   \n",
       "1    5cr7r_f0i_6wyd0xw3ayza:93764b25ae99d315f410802...     cancer   \n",
       "2    _hyofx-kvkxkzrumj0tvoa:a99dca5593185c498b63a5e...     cancer   \n",
       "3    _wbpom97rhxkmccuzyb26g:7463543d784aa59ca86359a...     cancer   \n",
       "4    aajrft_asz6yj4tjzrqkdq:26985a88de1e140c5bce6ac...     cancer   \n",
       "..                                                 ...        ...   \n",
       "434  zh8ka-z38c5-lt2u_es5pa:a99dca5593185c498b63a5e...     cancer   \n",
       "435  zixesdd3x1a8dq8qf40noa:bce3586a1d3a3cd33201d84...     cancer   \n",
       "436  zvydnkesoc5_lxcvysq50g:c2ba8420ea8e9b67e985e4d...     cancer   \n",
       "437  zx-dmpxxkv1fk8ustboiia:6a1cd9549015db5492d2984...     cancer   \n",
       "438  zzd1qvexmjxhr6bzsoar2w:a99dca5593185c498b63a5e...     cancer   \n",
       "\n",
       "                         filters sort_algorithm  result_count  page_num  \\\n",
       "0                           None      relevance       4569599         1   \n",
       "1                           None           date       4569599         1   \n",
       "2                           None      relevance       4569599         1   \n",
       "3                           None      relevance       4569599         1   \n",
       "4                 datesearch.y_5      relevance       1064948         1   \n",
       "..                           ...            ...           ...       ...   \n",
       "434                         None      relevance       4565187         1   \n",
       "435  pubt.review,lang.portuguese      relevance           801         1   \n",
       "436                         None      relevance       4565187         1   \n",
       "437             simsearch2.ffrft      relevance       1431624         1   \n",
       "438                         None      relevance       4565187         1   \n",
       "\n",
       "                                                  PMID  \\\n",
       "0    31761807,28244479,27741350,30713326,29949179,2...   \n",
       "1    35289556,35289548,35289536,35289535,35289519,3...   \n",
       "2    31761807,28244479,27741350,30713326,29949179,2...   \n",
       "3    31761807,28244479,27741350,30713326,29949179,2...   \n",
       "4    29860986,31761807,30445141,28154351,30713326,2...   \n",
       "..                                                 ...   \n",
       "434  31761807,28244479,27741350,30713326,29949179,2...   \n",
       "435  28614430,29160494,32354458,29538545,32667426,2...   \n",
       "436  31761807,28244479,27741350,30713326,29949179,2...   \n",
       "437  18538731,29860986,28244479,28154351,30713326,2...   \n",
       "438  31761807,28244479,27741350,30713326,29949179,2...   \n",
       "\n",
       "                                            click_data        date  user_id  \\\n",
       "0                                             NoClicks  2022-03-16  Z392386   \n",
       "1                                             NoClicks  2022-03-16  Z401427   \n",
       "2    *result_click,28244479,2022-03-16T20:03:49,1,F...  2022-03-16  Z272979   \n",
       "3                                             NoClicks  2022-03-16   Z93392   \n",
       "4    *result_click,31761807,2022-03-16T20:44:35,1,F...  2022-03-16  Z292543   \n",
       "..                                                 ...         ...      ...   \n",
       "434                                           NoClicks  2022-03-09  Z391060   \n",
       "435  *result_click,29538545,2022-03-09T09:04:47,1,F...  2022-03-09  Z141330   \n",
       "436                                           NoClicks  2022-03-09  Z201014   \n",
       "437                                           NoClicks  2022-03-09  Z381937   \n",
       "438                                           NoClicks  2022-03-09  Z237552   \n",
       "\n",
       "     filters_active processed_query  \n",
       "0             False          cancer  \n",
       "1             False          cancer  \n",
       "2             False          cancer  \n",
       "3             False          cancer  \n",
       "4              True          cancer  \n",
       "..              ...             ...  \n",
       "434           False          cancer  \n",
       "435            True          cancer  \n",
       "436           False          cancer  \n",
       "437            True          cancer  \n",
       "438           False          cancer  \n",
       "\n",
       "[439 rows x 12 columns]"
      ]
     },
     "execution_count": 5,
>>>>>>> da1efa8628124ffed82b84aafc58e2a7734db5bb
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
<<<<<<< HEAD
    "cancer_df.head()"
=======
    "cancer_df = pd.read_csv('/data/team2/top10data/cancer.csv')\n",
    "\n",
    "cancer_df"
>>>>>>> da1efa8628124ffed82b84aafc58e2a7734db5bb
   ]
  },
  {
   "cell_type": "code",
<<<<<<< HEAD
   "execution_count": 35,
   "id": "cd18faf5",
=======
   "execution_count": 6,
   "id": "bfe3761c",
>>>>>>> da1efa8628124ffed82b84aafc58e2a7734db5bb
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
<<<<<<< HEAD
       "      <th>Unnamed: 0</th>\n",
       "      <th>relevance_res</th>\n",
       "      <th>date_desc_res</th>\n",
       "      <th>query_type</th>\n",
=======
       "      <th>pmid</th>\n",
       "      <th>journal</th>\n",
       "      <th>title</th>\n",
       "      <th>abstract</th>\n",
       "      <th>hasAbstract</th>\n",
       "      <th>hasStructuredAbstract</th>\n",
       "      <th>articleDate</th>\n",
       "      <th>journalPubDate</th>\n",
       "      <th>entrezDate</th>\n",
       "      <th>pubtype</th>\n",
>>>>>>> da1efa8628124ffed82b84aafc58e2a7734db5bb
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
<<<<<<< HEAD
       "      <td>breast cancer</td>\n",
       "      <td>['29651637', '25456291', '31398351', '33786630...</td>\n",
       "      <td>['35612777', '35612761', '35612745', '35612727...</td>\n",
       "      <td>top_10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>alzheimer's disease</td>\n",
       "      <td>['27570871', '28269772', '27179961', '28211812...</td>\n",
       "      <td>['35612802', '35612780', '35612774', '35612546...</td>\n",
       "      <td>top_10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>cancer</td>\n",
       "      <td>['27839715', '31761807', '19491253', '26667886...</td>\n",
       "      <td>['35612878', '35612872', '35612856', '35612854...</td>\n",
       "      <td>top_10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>lung cancer</td>\n",
       "      <td>['34895608', '29157380', '28653990', '35263397...</td>\n",
       "      <td>['35612872', '35612853', '35612707', '35612630...</td>\n",
       "      <td>top_10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>obesity</td>\n",
       "      <td>['20871401', '15245383', '28292619', '7013664'...</td>\n",
       "      <td>['35612892', '35612619', '35612570', '35612502...</td>\n",
       "      <td>top_10</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            Unnamed: 0                                      relevance_res  \\\n",
       "0        breast cancer  ['29651637', '25456291', '31398351', '33786630...   \n",
       "1  alzheimer's disease  ['27570871', '28269772', '27179961', '28211812...   \n",
       "2               cancer  ['27839715', '31761807', '19491253', '26667886...   \n",
       "3          lung cancer  ['34895608', '29157380', '28653990', '35263397...   \n",
       "4              obesity  ['20871401', '15245383', '28292619', '7013664'...   \n",
       "\n",
       "                                       date_desc_res query_type  \n",
       "0  ['35612777', '35612761', '35612745', '35612727...     top_10  \n",
       "1  ['35612802', '35612780', '35612774', '35612546...     top_10  \n",
       "2  ['35612878', '35612872', '35612856', '35612854...     top_10  \n",
       "3  ['35612872', '35612853', '35612707', '35612630...     top_10  \n",
       "4  ['35612892', '35612619', '35612570', '35612502...     top_10  "
      ]
     },
     "execution_count": 35,
=======
       "      <td>35252802</td>\n",
       "      <td>Journal of biosafety and biosecurity</td>\n",
       "      <td>Recent developments and trends of automatic nu...</td>\n",
       "      <td>Nucleic acid detection, widely used in clinica...</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>2022-02-28</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2022-03-07</td>\n",
       "      <td>['Journal Article', 'Review']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>35287656</td>\n",
       "      <td>BMC public health</td>\n",
       "      <td>Nutrition standards for the charitable food sy...</td>\n",
       "      <td>Food insecurity is associated with increased r...</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>2022-03-14</td>\n",
       "      <td>2022-03-14</td>\n",
       "      <td>2022-03-15</td>\n",
       "      <td>['Journal Article', \"Research Support, Non-U.S...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>35232441</td>\n",
       "      <td>BMC medical education</td>\n",
       "      <td>Establishing CREATE: lessons learned in settin...</td>\n",
       "      <td>[{'@Label': 'BACKGROUND', '@NlmCategory': 'BAC...</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>2022-03-01</td>\n",
       "      <td>2022-03-01</td>\n",
       "      <td>2022-03-02</td>\n",
       "      <td>['Journal Article']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>32282054</td>\n",
       "      <td>Turk patoloji dergisi</td>\n",
       "      <td>Pulmonary Acantholytic Squamous Cell Carcinoma...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2020-04-14</td>\n",
       "      <td>['Journal Article']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>33629745</td>\n",
       "      <td>Journal of cellular physiology</td>\n",
       "      <td>TRIM11 suppresses ferritinophagy and gemcitabi...</td>\n",
       "      <td>Gemcitabine is first-line chemotherapy for pan...</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>2021-02-25</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2021-02-25</td>\n",
       "      <td>['Journal Article', \"Research Support, Non-U.S...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16814</th>\n",
       "      <td>34281775</td>\n",
       "      <td>Phytomedicine : international journal of phyto...</td>\n",
       "      <td>Identification of prenylated phenolics in mulb...</td>\n",
       "      <td>[{'@Label': 'BACKGROUND', '@NlmCategory': 'BAC...</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>2021-07-03</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2021-07-20</td>\n",
       "      <td>['Journal Article']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16815</th>\n",
       "      <td>33762716</td>\n",
       "      <td>British journal of cancer</td>\n",
       "      <td>Conventional versus reverse sequence of neoadj...</td>\n",
       "      <td>[{'@Label': 'BACKGROUND', '#text': 'Preoperati...</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>2021-03-24</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2021-03-25</td>\n",
       "      <td>['Clinical Trial, Phase II', 'Journal Article'...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16816</th>\n",
       "      <td>35239864</td>\n",
       "      <td>Revista do Instituto de Medicina Tropical de S...</td>\n",
       "      <td>Adult patients admitted to a tertiary hospital...</td>\n",
       "      <td>COVID-19 is a disease whose knowledge is still...</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>2022-02-25</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2022-03-03</td>\n",
       "      <td>['Journal Article', 'Observational Study']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16817</th>\n",
       "      <td>35163783</td>\n",
       "      <td>International journal of molecular sciences</td>\n",
       "      <td>Theranostic Interpolation of Genomic Instabili...</td>\n",
       "      <td>Breast cancer is a diverse disease caused by m...</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>2022-02-07</td>\n",
       "      <td>2022-02-07</td>\n",
       "      <td>2022-02-15</td>\n",
       "      <td>['Journal Article', 'Review']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16818</th>\n",
       "      <td>32835615</td>\n",
       "      <td>Journal of biomolecular structure &amp; dynamics</td>\n",
       "      <td>A database of integrated molecular and phytoch...</td>\n",
       "      <td>The FoxM1 pathway is an oncogenic signaling pa...</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>2020-08-24</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2020-08-25</td>\n",
       "      <td>['Journal Article']</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>16819 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           pmid                                            journal  \\\n",
       "0      35252802               Journal of biosafety and biosecurity   \n",
       "1      35287656                                  BMC public health   \n",
       "2      35232441                              BMC medical education   \n",
       "3      32282054                              Turk patoloji dergisi   \n",
       "4      33629745                     Journal of cellular physiology   \n",
       "...         ...                                                ...   \n",
       "16814  34281775  Phytomedicine : international journal of phyto...   \n",
       "16815  33762716                          British journal of cancer   \n",
       "16816  35239864  Revista do Instituto de Medicina Tropical de S...   \n",
       "16817  35163783        International journal of molecular sciences   \n",
       "16818  32835615       Journal of biomolecular structure & dynamics   \n",
       "\n",
       "                                                   title  \\\n",
       "0      Recent developments and trends of automatic nu...   \n",
       "1      Nutrition standards for the charitable food sy...   \n",
       "2      Establishing CREATE: lessons learned in settin...   \n",
       "3      Pulmonary Acantholytic Squamous Cell Carcinoma...   \n",
       "4      TRIM11 suppresses ferritinophagy and gemcitabi...   \n",
       "...                                                  ...   \n",
       "16814  Identification of prenylated phenolics in mulb...   \n",
       "16815  Conventional versus reverse sequence of neoadj...   \n",
       "16816  Adult patients admitted to a tertiary hospital...   \n",
       "16817  Theranostic Interpolation of Genomic Instabili...   \n",
       "16818  A database of integrated molecular and phytoch...   \n",
       "\n",
       "                                                abstract  hasAbstract  \\\n",
       "0      Nucleic acid detection, widely used in clinica...         True   \n",
       "1      Food insecurity is associated with increased r...         True   \n",
       "2      [{'@Label': 'BACKGROUND', '@NlmCategory': 'BAC...         True   \n",
       "3                                                    NaN        False   \n",
       "4      Gemcitabine is first-line chemotherapy for pan...         True   \n",
       "...                                                  ...          ...   \n",
       "16814  [{'@Label': 'BACKGROUND', '@NlmCategory': 'BAC...         True   \n",
       "16815  [{'@Label': 'BACKGROUND', '#text': 'Preoperati...         True   \n",
       "16816  COVID-19 is a disease whose knowledge is still...         True   \n",
       "16817  Breast cancer is a diverse disease caused by m...         True   \n",
       "16818  The FoxM1 pathway is an oncogenic signaling pa...         True   \n",
       "\n",
       "       hasStructuredAbstract articleDate journalPubDate  entrezDate  \\\n",
       "0                      False  2022-02-28            NaN  2022-03-07   \n",
       "1                      False  2022-03-14     2022-03-14  2022-03-15   \n",
       "2                       True  2022-03-01     2022-03-01  2022-03-02   \n",
       "3                      False         NaN            NaN  2020-04-14   \n",
       "4                      False  2021-02-25            NaN  2021-02-25   \n",
       "...                      ...         ...            ...         ...   \n",
       "16814                   True  2021-07-03            NaN  2021-07-20   \n",
       "16815                   True  2021-03-24            NaN  2021-03-25   \n",
       "16816                  False  2022-02-25            NaN  2022-03-03   \n",
       "16817                  False  2022-02-07     2022-02-07  2022-02-15   \n",
       "16818                  False  2020-08-24            NaN  2020-08-25   \n",
       "\n",
       "                                                 pubtype  \n",
       "0                          ['Journal Article', 'Review']  \n",
       "1      ['Journal Article', \"Research Support, Non-U.S...  \n",
       "2                                    ['Journal Article']  \n",
       "3                                    ['Journal Article']  \n",
       "4      ['Journal Article', \"Research Support, Non-U.S...  \n",
       "...                                                  ...  \n",
       "16814                                ['Journal Article']  \n",
       "16815  ['Clinical Trial, Phase II', 'Journal Article'...  \n",
       "16816         ['Journal Article', 'Observational Study']  \n",
       "16817                      ['Journal Article', 'Review']  \n",
       "16818                                ['Journal Article']  \n",
       "\n",
       "[16819 rows x 10 columns]"
      ]
     },
     "execution_count": 6,
>>>>>>> da1efa8628124ffed82b84aafc58e2a7734db5bb
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
<<<<<<< HEAD
    "api_res.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "3418d1f1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "breast cancer\n",
      "117227\n",
      "119988\n",
      "\n",
      "\n",
      "alzheimer's disease\n",
      "117719\n",
      "119988\n",
      "\n",
      "\n",
      "cancer\n",
      "116963\n",
      "119988\n",
      "\n",
      "\n",
      "lung cancer\n",
      "116731\n",
      "119988\n",
      "\n",
      "\n",
      "obesity\n",
      "117368\n",
      "119988\n",
      "\n",
      "\n",
      "multiple sclerosis\n",
      "117606\n",
      "119988\n",
      "\n",
      "\n",
      "diabetes mellitus\n",
      "114103\n",
      "119988\n",
      "\n",
      "\n",
      "gastric cancer\n",
      "115903\n",
      "119988\n",
      "\n",
      "\n",
      "ferroptosis\n",
      "49080\n",
      "49080\n",
      "\n",
      "\n",
      "covid-19\n",
      "119988\n",
      "119988\n",
      "\n",
      "\n",
      "allergies\n",
      "116800\n",
      "119988\n",
      "\n",
      "\n",
      "cardiac ischemia\n",
      "114415\n",
      "119988\n",
      "\n",
      "\n",
      "neurodegenerative disease\n",
      "119709\n",
      "119988\n",
      "\n",
      "\n",
      "depression in older adults\n",
      "5078\n",
      "5078\n",
      "\n",
      "\n",
      "adverse drug reactions\n",
      "117362\n",
      "119988\n",
      "\n",
      "\n",
      "neuroinflammation\n",
      "117402\n",
      "119988\n",
      "\n",
      "\n",
      "vasculitis\n",
      "116335\n",
      "119988\n",
      "\n",
      "\n",
      "hypoxia\n",
      "114963\n",
      "119988\n",
      "\n",
      "\n",
      "acute kidney injury\n",
      "116324\n",
      "119988\n",
      "\n",
      "\n",
      "pulmonary embolism\n",
      "114949\n",
      "119988\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for i in range(api_res.shape[0]):\n",
    "    print(api_res.iloc[i,0])\n",
    "    print(len(api_res.iloc[i,1]))\n",
    "    print(len(api_res.iloc[i,2]))\n",
    "    print('\\n')"
=======
    "top_10_query_PMIDs = pd.read_csv('/data/team2/top_10_query_PMIDs_v2.csv').drop('Unnamed: 0', axis=1)\n",
    "top_10_query_PMIDs"
>>>>>>> da1efa8628124ffed82b84aafc58e2a7734db5bb
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
<<<<<<< HEAD
   "id": "745f89c3",
=======
   "id": "c847d528",
>>>>>>> da1efa8628124ffed82b84aafc58e2a7734db5bb
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}