flowcyto_create_traintest_datasets.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create the train and test datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>FSC-A</th>\n",
       "      <th>FSC-H</th>\n",
       "      <th>FSC-W</th>\n",
       "      <th>SSC-A</th>\n",
       "      <th>SSC-H</th>\n",
       "      <th>SSC-W</th>\n",
       "      <th>Comp-Alexa Fluor 700-A :: CD4</th>\n",
       "      <th>Comp-BUV737-A :: CD5</th>\n",
       "      <th>Comp-Brilliant Violet 421-A :: CD107a</th>\n",
       "      <th>Comp-Brilliant Violet 605-A :: CD8</th>\n",
       "      <th>Comp-FITC-A :: CD226</th>\n",
       "      <th>Comp-LIVE_DEAD Fixable Blue-A</th>\n",
       "      <th>Comp-PE-A :: TIGIT</th>\n",
       "      <th>Comp-PE-CF594-A :: CD197</th>\n",
       "      <th>Comp-PE-Cy5-A :: CD28</th>\n",
       "      <th>Comp-PE-Cy7-A :: CD45RA</th>\n",
       "      <th>Time :: CD226</th>\n",
       "      <th>class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>321</td>\n",
       "      <td>316</td>\n",
       "      <td>260</td>\n",
       "      <td>216</td>\n",
       "      <td>285</td>\n",
       "      <td>194</td>\n",
       "      <td>557</td>\n",
       "      <td>734</td>\n",
       "      <td>809</td>\n",
       "      <td>272</td>\n",
       "      <td>378</td>\n",
       "      <td>390</td>\n",
       "      <td>663</td>\n",
       "      <td>510</td>\n",
       "      <td>664</td>\n",
       "      <td>205</td>\n",
       "      <td>0</td>\n",
       "      <td>CD4_tcm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>335</td>\n",
       "      <td>288</td>\n",
       "      <td>298</td>\n",
       "      <td>138</td>\n",
       "      <td>274</td>\n",
       "      <td>129</td>\n",
       "      <td>486</td>\n",
       "      <td>487</td>\n",
       "      <td>807</td>\n",
       "      <td>258</td>\n",
       "      <td>277</td>\n",
       "      <td>307</td>\n",
       "      <td>236</td>\n",
       "      <td>395</td>\n",
       "      <td>523</td>\n",
       "      <td>626</td>\n",
       "      <td>1</td>\n",
       "      <td>CD4_tcm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>336</td>\n",
       "      <td>304</td>\n",
       "      <td>283</td>\n",
       "      <td>174</td>\n",
       "      <td>270</td>\n",
       "      <td>165</td>\n",
       "      <td>591</td>\n",
       "      <td>609</td>\n",
       "      <td>806</td>\n",
       "      <td>232</td>\n",
       "      <td>317</td>\n",
       "      <td>389</td>\n",
       "      <td>611</td>\n",
       "      <td>421</td>\n",
       "      <td>500</td>\n",
       "      <td>249</td>\n",
       "      <td>3</td>\n",
       "      <td>CD4_tcm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>295</td>\n",
       "      <td>276</td>\n",
       "      <td>274</td>\n",
       "      <td>83</td>\n",
       "      <td>263</td>\n",
       "      <td>81</td>\n",
       "      <td>519</td>\n",
       "      <td>557</td>\n",
       "      <td>638</td>\n",
       "      <td>259</td>\n",
       "      <td>250</td>\n",
       "      <td>282</td>\n",
       "      <td>254</td>\n",
       "      <td>492</td>\n",
       "      <td>466</td>\n",
       "      <td>638</td>\n",
       "      <td>4</td>\n",
       "      <td>CD4_tcm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>307</td>\n",
       "      <td>293</td>\n",
       "      <td>268</td>\n",
       "      <td>161</td>\n",
       "      <td>275</td>\n",
       "      <td>150</td>\n",
       "      <td>575</td>\n",
       "      <td>575</td>\n",
       "      <td>792</td>\n",
       "      <td>289</td>\n",
       "      <td>273</td>\n",
       "      <td>386</td>\n",
       "      <td>189</td>\n",
       "      <td>405</td>\n",
       "      <td>583</td>\n",
       "      <td>561</td>\n",
       "      <td>5</td>\n",
       "      <td>CD4_tcm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6807425</th>\n",
       "      <td>393</td>\n",
       "      <td>379</td>\n",
       "      <td>266</td>\n",
       "      <td>308</td>\n",
       "      <td>297</td>\n",
       "      <td>266</td>\n",
       "      <td>321</td>\n",
       "      <td>269</td>\n",
       "      <td>284</td>\n",
       "      <td>296</td>\n",
       "      <td>298</td>\n",
       "      <td>382</td>\n",
       "      <td>249</td>\n",
       "      <td>359</td>\n",
       "      <td>234</td>\n",
       "      <td>333</td>\n",
       "      <td>1023</td>\n",
       "      <td>debris</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6807426</th>\n",
       "      <td>232</td>\n",
       "      <td>273</td>\n",
       "      <td>217</td>\n",
       "      <td>285</td>\n",
       "      <td>265</td>\n",
       "      <td>275</td>\n",
       "      <td>510</td>\n",
       "      <td>363</td>\n",
       "      <td>215</td>\n",
       "      <td>277</td>\n",
       "      <td>273</td>\n",
       "      <td>562</td>\n",
       "      <td>277</td>\n",
       "      <td>216</td>\n",
       "      <td>264</td>\n",
       "      <td>231</td>\n",
       "      <td>1023</td>\n",
       "      <td>debris</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6807427</th>\n",
       "      <td>271</td>\n",
       "      <td>263</td>\n",
       "      <td>265</td>\n",
       "      <td>210</td>\n",
       "      <td>264</td>\n",
       "      <td>203</td>\n",
       "      <td>104</td>\n",
       "      <td>612</td>\n",
       "      <td>67</td>\n",
       "      <td>304</td>\n",
       "      <td>278</td>\n",
       "      <td>500</td>\n",
       "      <td>166</td>\n",
       "      <td>263</td>\n",
       "      <td>294</td>\n",
       "      <td>804</td>\n",
       "      <td>1023</td>\n",
       "      <td>debris</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6807428</th>\n",
       "      <td>280</td>\n",
       "      <td>271</td>\n",
       "      <td>264</td>\n",
       "      <td>170</td>\n",
       "      <td>263</td>\n",
       "      <td>166</td>\n",
       "      <td>237</td>\n",
       "      <td>225</td>\n",
       "      <td>258</td>\n",
       "      <td>251</td>\n",
       "      <td>254</td>\n",
       "      <td>283</td>\n",
       "      <td>242</td>\n",
       "      <td>184</td>\n",
       "      <td>229</td>\n",
       "      <td>221</td>\n",
       "      <td>1023</td>\n",
       "      <td>debris</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6807429</th>\n",
       "      <td>253</td>\n",
       "      <td>261</td>\n",
       "      <td>249</td>\n",
       "      <td>142</td>\n",
       "      <td>259</td>\n",
       "      <td>141</td>\n",
       "      <td>118</td>\n",
       "      <td>612</td>\n",
       "      <td>127</td>\n",
       "      <td>241</td>\n",
       "      <td>249</td>\n",
       "      <td>453</td>\n",
       "      <td>300</td>\n",
       "      <td>227</td>\n",
       "      <td>241</td>\n",
       "      <td>374</td>\n",
       "      <td>1023</td>\n",
       "      <td>debris</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>6807430 rows × 18 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         FSC-A  FSC-H  FSC-W  SSC-A  SSC-H  SSC-W  \\\n",
       "0          321    316    260    216    285    194   \n",
       "1          335    288    298    138    274    129   \n",
       "2          336    304    283    174    270    165   \n",
       "3          295    276    274     83    263     81   \n",
       "4          307    293    268    161    275    150   \n",
       "...        ...    ...    ...    ...    ...    ...   \n",
       "6807425    393    379    266    308    297    266   \n",
       "6807426    232    273    217    285    265    275   \n",
       "6807427    271    263    265    210    264    203   \n",
       "6807428    280    271    264    170    263    166   \n",
       "6807429    253    261    249    142    259    141   \n",
       "\n",
       "         Comp-Alexa Fluor 700-A :: CD4  Comp-BUV737-A :: CD5  \\\n",
       "0                                  557                   734   \n",
       "1                                  486                   487   \n",
       "2                                  591                   609   \n",
       "3                                  519                   557   \n",
       "4                                  575                   575   \n",
       "...                                ...                   ...   \n",
       "6807425                            321                   269   \n",
       "6807426                            510                   363   \n",
       "6807427                            104                   612   \n",
       "6807428                            237                   225   \n",
       "6807429                            118                   612   \n",
       "\n",
       "         Comp-Brilliant Violet 421-A :: CD107a  \\\n",
       "0                                          809   \n",
       "1                                          807   \n",
       "2                                          806   \n",
       "3                                          638   \n",
       "4                                          792   \n",
       "...                                        ...   \n",
       "6807425                                    284   \n",
       "6807426                                    215   \n",
       "6807427                                     67   \n",
       "6807428                                    258   \n",
       "6807429                                    127   \n",
       "\n",
       "         Comp-Brilliant Violet 605-A :: CD8  Comp-FITC-A :: CD226  \\\n",
       "0                                       272                   378   \n",
       "1                                       258                   277   \n",
       "2                                       232                   317   \n",
       "3                                       259                   250   \n",
       "4                                       289                   273   \n",
       "...                                     ...                   ...   \n",
       "6807425                                 296                   298   \n",
       "6807426                                 277                   273   \n",
       "6807427                                 304                   278   \n",
       "6807428                                 251                   254   \n",
       "6807429                                 241                   249   \n",
       "\n",
       "         Comp-LIVE_DEAD Fixable Blue-A  Comp-PE-A :: TIGIT  \\\n",
       "0                                  390                 663   \n",
       "1                                  307                 236   \n",
       "2                                  389                 611   \n",
       "3                                  282                 254   \n",
       "4                                  386                 189   \n",
       "...                                ...                 ...   \n",
       "6807425                            382                 249   \n",
       "6807426                            562                 277   \n",
       "6807427                            500                 166   \n",
       "6807428                            283                 242   \n",
       "6807429                            453                 300   \n",
       "\n",
       "         Comp-PE-CF594-A :: CD197  Comp-PE-Cy5-A :: CD28  \\\n",
       "0                             510                    664   \n",
       "1                             395                    523   \n",
       "2                             421                    500   \n",
       "3                             492                    466   \n",
       "4                             405                    583   \n",
       "...                           ...                    ...   \n",
       "6807425                       359                    234   \n",
       "6807426                       216                    264   \n",
       "6807427                       263                    294   \n",
       "6807428                       184                    229   \n",
       "6807429                       227                    241   \n",
       "\n",
       "         Comp-PE-Cy7-A :: CD45RA  Time :: CD226    class  \n",
       "0                            205              0  CD4_tcm  \n",
       "1                            626              1  CD4_tcm  \n",
       "2                            249              3  CD4_tcm  \n",
       "3                            638              4  CD4_tcm  \n",
       "4                            561              5  CD4_tcm  \n",
       "...                          ...            ...      ...  \n",
       "6807425                      333           1023   debris  \n",
       "6807426                      231           1023   debris  \n",
       "6807427                      804           1023   debris  \n",
       "6807428                      221           1023   debris  \n",
       "6807429                      374           1023   debris  \n",
       "\n",
       "[6807430 rows x 18 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "total_grouped = pd.read_csv(\"data/total_grouped.csv\")\n",
    "total_grouped"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Clean up column names:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>FSC_A</th>\n",
       "      <th>FSC_H</th>\n",
       "      <th>FSC_W</th>\n",
       "      <th>SSC_A</th>\n",
       "      <th>SSC_H</th>\n",
       "      <th>SSC_W</th>\n",
       "      <th>CD4</th>\n",
       "      <th>CD5</th>\n",
       "      <th>CD107a</th>\n",
       "      <th>CD8</th>\n",
       "      <th>CD226</th>\n",
       "      <th>LIVE_DEAD</th>\n",
       "      <th>TIGIT</th>\n",
       "      <th>CD197</th>\n",
       "      <th>CD28</th>\n",
       "      <th>CD45RA</th>\n",
       "      <th>time</th>\n",
       "      <th>class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>321</td>\n",
       "      <td>316</td>\n",
       "      <td>260</td>\n",
       "      <td>216</td>\n",
       "      <td>285</td>\n",
       "      <td>194</td>\n",
       "      <td>557</td>\n",
       "      <td>734</td>\n",
       "      <td>809</td>\n",
       "      <td>272</td>\n",
       "      <td>378</td>\n",
       "      <td>390</td>\n",
       "      <td>663</td>\n",
       "      <td>510</td>\n",
       "      <td>664</td>\n",
       "      <td>205</td>\n",
       "      <td>0</td>\n",
       "      <td>CD4_tcm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>335</td>\n",
       "      <td>288</td>\n",
       "      <td>298</td>\n",
       "      <td>138</td>\n",
       "      <td>274</td>\n",
       "      <td>129</td>\n",
       "      <td>486</td>\n",
       "      <td>487</td>\n",
       "      <td>807</td>\n",
       "      <td>258</td>\n",
       "      <td>277</td>\n",
       "      <td>307</td>\n",
       "      <td>236</td>\n",
       "      <td>395</td>\n",
       "      <td>523</td>\n",
       "      <td>626</td>\n",
       "      <td>1</td>\n",
       "      <td>CD4_tcm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>336</td>\n",
       "      <td>304</td>\n",
       "      <td>283</td>\n",
       "      <td>174</td>\n",
       "      <td>270</td>\n",
       "      <td>165</td>\n",
       "      <td>591</td>\n",
       "      <td>609</td>\n",
       "      <td>806</td>\n",
       "      <td>232</td>\n",
       "      <td>317</td>\n",
       "      <td>389</td>\n",
       "      <td>611</td>\n",
       "      <td>421</td>\n",
       "      <td>500</td>\n",
       "      <td>249</td>\n",
       "      <td>3</td>\n",
       "      <td>CD4_tcm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>295</td>\n",
       "      <td>276</td>\n",
       "      <td>274</td>\n",
       "      <td>83</td>\n",
       "      <td>263</td>\n",
       "      <td>81</td>\n",
       "      <td>519</td>\n",
       "      <td>557</td>\n",
       "      <td>638</td>\n",
       "      <td>259</td>\n",
       "      <td>250</td>\n",
       "      <td>282</td>\n",
       "      <td>254</td>\n",
       "      <td>492</td>\n",
       "      <td>466</td>\n",
       "      <td>638</td>\n",
       "      <td>4</td>\n",
       "      <td>CD4_tcm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>307</td>\n",
       "      <td>293</td>\n",
       "      <td>268</td>\n",
       "      <td>161</td>\n",
       "      <td>275</td>\n",
       "      <td>150</td>\n",
       "      <td>575</td>\n",
       "      <td>575</td>\n",
       "      <td>792</td>\n",
       "      <td>289</td>\n",
       "      <td>273</td>\n",
       "      <td>386</td>\n",
       "      <td>189</td>\n",
       "      <td>405</td>\n",
       "      <td>583</td>\n",
       "      <td>561</td>\n",
       "      <td>5</td>\n",
       "      <td>CD4_tcm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6807425</th>\n",
       "      <td>393</td>\n",
       "      <td>379</td>\n",
       "      <td>266</td>\n",
       "      <td>308</td>\n",
       "      <td>297</td>\n",
       "      <td>266</td>\n",
       "      <td>321</td>\n",
       "      <td>269</td>\n",
       "      <td>284</td>\n",
       "      <td>296</td>\n",
       "      <td>298</td>\n",
       "      <td>382</td>\n",
       "      <td>249</td>\n",
       "      <td>359</td>\n",
       "      <td>234</td>\n",
       "      <td>333</td>\n",
       "      <td>1023</td>\n",
       "      <td>debris</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6807426</th>\n",
       "      <td>232</td>\n",
       "      <td>273</td>\n",
       "      <td>217</td>\n",
       "      <td>285</td>\n",
       "      <td>265</td>\n",
       "      <td>275</td>\n",
       "      <td>510</td>\n",
       "      <td>363</td>\n",
       "      <td>215</td>\n",
       "      <td>277</td>\n",
       "      <td>273</td>\n",
       "      <td>562</td>\n",
       "      <td>277</td>\n",
       "      <td>216</td>\n",
       "      <td>264</td>\n",
       "      <td>231</td>\n",
       "      <td>1023</td>\n",
       "      <td>debris</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6807427</th>\n",
       "      <td>271</td>\n",
       "      <td>263</td>\n",
       "      <td>265</td>\n",
       "      <td>210</td>\n",
       "      <td>264</td>\n",
       "      <td>203</td>\n",
       "      <td>104</td>\n",
       "      <td>612</td>\n",
       "      <td>67</td>\n",
       "      <td>304</td>\n",
       "      <td>278</td>\n",
       "      <td>500</td>\n",
       "      <td>166</td>\n",
       "      <td>263</td>\n",
       "      <td>294</td>\n",
       "      <td>804</td>\n",
       "      <td>1023</td>\n",
       "      <td>debris</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6807428</th>\n",
       "      <td>280</td>\n",
       "      <td>271</td>\n",
       "      <td>264</td>\n",
       "      <td>170</td>\n",
       "      <td>263</td>\n",
       "      <td>166</td>\n",
       "      <td>237</td>\n",
       "      <td>225</td>\n",
       "      <td>258</td>\n",
       "      <td>251</td>\n",
       "      <td>254</td>\n",
       "      <td>283</td>\n",
       "      <td>242</td>\n",
       "      <td>184</td>\n",
       "      <td>229</td>\n",
       "      <td>221</td>\n",
       "      <td>1023</td>\n",
       "      <td>debris</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6807429</th>\n",
       "      <td>253</td>\n",
       "      <td>261</td>\n",
       "      <td>249</td>\n",
       "      <td>142</td>\n",
       "      <td>259</td>\n",
       "      <td>141</td>\n",
       "      <td>118</td>\n",
       "      <td>612</td>\n",
       "      <td>127</td>\n",
       "      <td>241</td>\n",
       "      <td>249</td>\n",
       "      <td>453</td>\n",
       "      <td>300</td>\n",
       "      <td>227</td>\n",
       "      <td>241</td>\n",
       "      <td>374</td>\n",
       "      <td>1023</td>\n",
       "      <td>debris</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>6807430 rows × 18 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         FSC_A  FSC_H  FSC_W  SSC_A  SSC_H  SSC_W  CD4  CD5  CD107a  CD8  \\\n",
       "0          321    316    260    216    285    194  557  734     809  272   \n",
       "1          335    288    298    138    274    129  486  487     807  258   \n",
       "2          336    304    283    174    270    165  591  609     806  232   \n",
       "3          295    276    274     83    263     81  519  557     638  259   \n",
       "4          307    293    268    161    275    150  575  575     792  289   \n",
       "...        ...    ...    ...    ...    ...    ...  ...  ...     ...  ...   \n",
       "6807425    393    379    266    308    297    266  321  269     284  296   \n",
       "6807426    232    273    217    285    265    275  510  363     215  277   \n",
       "6807427    271    263    265    210    264    203  104  612      67  304   \n",
       "6807428    280    271    264    170    263    166  237  225     258  251   \n",
       "6807429    253    261    249    142    259    141  118  612     127  241   \n",
       "\n",
       "         CD226  LIVE_DEAD  TIGIT  CD197  CD28  CD45RA  time    class  \n",
       "0          378        390    663    510   664     205     0  CD4_tcm  \n",
       "1          277        307    236    395   523     626     1  CD4_tcm  \n",
       "2          317        389    611    421   500     249     3  CD4_tcm  \n",
       "3          250        282    254    492   466     638     4  CD4_tcm  \n",
       "4          273        386    189    405   583     561     5  CD4_tcm  \n",
       "...        ...        ...    ...    ...   ...     ...   ...      ...  \n",
       "6807425    298        382    249    359   234     333  1023   debris  \n",
       "6807426    273        562    277    216   264     231  1023   debris  \n",
       "6807427    278        500    166    263   294     804  1023   debris  \n",
       "6807428    254        283    242    184   229     221  1023   debris  \n",
       "6807429    249        453    300    227   241     374  1023   debris  \n",
       "\n",
       "[6807430 rows x 18 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "total_grouped = total_grouped.rename(columns={\n",
    "                              \"FSC-A\":\"FSC_A\",\n",
    "                              \"FSC-H\":\"FSC_H\",\n",
    "                              \"FSC-W\":\"FSC_W\",\n",
    "                              \"SSC-A\":\"SSC_A\",\n",
    "                              \"SSC-H\":\"SSC_H\",\n",
    "                              \"SSC-W\":\"SSC_W\",\n",
    "                              \"Comp-Alexa Fluor 700-A :: CD4\":\"CD4\",\n",
    "                              \"Comp-BUV737-A :: CD5\":\"CD5\",\n",
    "                              \"Comp-Brilliant Violet 421-A :: CD107a\":\"CD107a\",\n",
    "                              \"Comp-Brilliant Violet 605-A :: CD8\":\"CD8\",\n",
    "                              \"Comp-FITC-A :: CD226\":\"CD226\",\n",
    "                              \"Comp-LIVE_DEAD Fixable Blue-A\":\"LIVE_DEAD\",\n",
    "                              \"Comp-PE-A :: TIGIT\":\"TIGIT\",\n",
    "                              \"Comp-PE-CF594-A :: CD197\":\"CD197\",\n",
    "                              \"Comp-PE-Cy5-A :: CD28\":\"CD28\",\n",
    "                              \"Comp-PE-Cy7-A :: CD45RA\":\"CD45RA\",\n",
    "                              \"Time :: CD226\":\"time\", \n",
    "                             })\n",
    "total_grouped"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Remove the CD107a, CD226, TIGIT and CD28 markers as they are expected not to influence the CD4 and CD8 types used in this model. Which simplifies the model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>FSC_A</th>\n",
       "      <th>FSC_H</th>\n",
       "      <th>FSC_W</th>\n",
       "      <th>SSC_A</th>\n",
       "      <th>SSC_H</th>\n",
       "      <th>SSC_W</th>\n",
       "      <th>CD4</th>\n",
       "      <th>CD5</th>\n",
       "      <th>CD8</th>\n",
       "      <th>LIVE_DEAD</th>\n",
       "      <th>CD197</th>\n",
       "      <th>CD45RA</th>\n",
       "      <th>class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>321</td>\n",
       "      <td>316</td>\n",
       "      <td>260</td>\n",
       "      <td>216</td>\n",
       "      <td>285</td>\n",
       "      <td>194</td>\n",
       "      <td>557</td>\n",
       "      <td>734</td>\n",
       "      <td>272</td>\n",
       "      <td>390</td>\n",
       "      <td>510</td>\n",
       "      <td>205</td>\n",
       "      <td>CD4_tcm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>335</td>\n",
       "      <td>288</td>\n",
       "      <td>298</td>\n",
       "      <td>138</td>\n",
       "      <td>274</td>\n",
       "      <td>129</td>\n",
       "      <td>486</td>\n",
       "      <td>487</td>\n",
       "      <td>258</td>\n",
       "      <td>307</td>\n",
       "      <td>395</td>\n",
       "      <td>626</td>\n",
       "      <td>CD4_tcm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>336</td>\n",
       "      <td>304</td>\n",
       "      <td>283</td>\n",
       "      <td>174</td>\n",
       "      <td>270</td>\n",
       "      <td>165</td>\n",
       "      <td>591</td>\n",
       "      <td>609</td>\n",
       "      <td>232</td>\n",
       "      <td>389</td>\n",
       "      <td>421</td>\n",
       "      <td>249</td>\n",
       "      <td>CD4_tcm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>295</td>\n",
       "      <td>276</td>\n",
       "      <td>274</td>\n",
       "      <td>83</td>\n",
       "      <td>263</td>\n",
       "      <td>81</td>\n",
       "      <td>519</td>\n",
       "      <td>557</td>\n",
       "      <td>259</td>\n",
       "      <td>282</td>\n",
       "      <td>492</td>\n",
       "      <td>638</td>\n",
       "      <td>CD4_tcm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>307</td>\n",
       "      <td>293</td>\n",
       "      <td>268</td>\n",
       "      <td>161</td>\n",
       "      <td>275</td>\n",
       "      <td>150</td>\n",
       "      <td>575</td>\n",
       "      <td>575</td>\n",
       "      <td>289</td>\n",
       "      <td>386</td>\n",
       "      <td>405</td>\n",
       "      <td>561</td>\n",
       "      <td>CD4_tcm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6807425</th>\n",
       "      <td>393</td>\n",
       "      <td>379</td>\n",
       "      <td>266</td>\n",
       "      <td>308</td>\n",
       "      <td>297</td>\n",
       "      <td>266</td>\n",
       "      <td>321</td>\n",
       "      <td>269</td>\n",
       "      <td>296</td>\n",
       "      <td>382</td>\n",
       "      <td>359</td>\n",
       "      <td>333</td>\n",
       "      <td>debris</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6807426</th>\n",
       "      <td>232</td>\n",
       "      <td>273</td>\n",
       "      <td>217</td>\n",
       "      <td>285</td>\n",
       "      <td>265</td>\n",
       "      <td>275</td>\n",
       "      <td>510</td>\n",
       "      <td>363</td>\n",
       "      <td>277</td>\n",
       "      <td>562</td>\n",
       "      <td>216</td>\n",
       "      <td>231</td>\n",
       "      <td>debris</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6807427</th>\n",
       "      <td>271</td>\n",
       "      <td>263</td>\n",
       "      <td>265</td>\n",
       "      <td>210</td>\n",
       "      <td>264</td>\n",
       "      <td>203</td>\n",
       "      <td>104</td>\n",
       "      <td>612</td>\n",
       "      <td>304</td>\n",
       "      <td>500</td>\n",
       "      <td>263</td>\n",
       "      <td>804</td>\n",
       "      <td>debris</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6807428</th>\n",
       "      <td>280</td>\n",
       "      <td>271</td>\n",
       "      <td>264</td>\n",
       "      <td>170</td>\n",
       "      <td>263</td>\n",
       "      <td>166</td>\n",
       "      <td>237</td>\n",
       "      <td>225</td>\n",
       "      <td>251</td>\n",
       "      <td>283</td>\n",
       "      <td>184</td>\n",
       "      <td>221</td>\n",
       "      <td>debris</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6807429</th>\n",
       "      <td>253</td>\n",
       "      <td>261</td>\n",
       "      <td>249</td>\n",
       "      <td>142</td>\n",
       "      <td>259</td>\n",
       "      <td>141</td>\n",
       "      <td>118</td>\n",
       "      <td>612</td>\n",
       "      <td>241</td>\n",
       "      <td>453</td>\n",
       "      <td>227</td>\n",
       "      <td>374</td>\n",
       "      <td>debris</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>6807430 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         FSC_A  FSC_H  FSC_W  SSC_A  SSC_H  SSC_W  CD4  CD5  CD8  LIVE_DEAD  \\\n",
       "0          321    316    260    216    285    194  557  734  272        390   \n",
       "1          335    288    298    138    274    129  486  487  258        307   \n",
       "2          336    304    283    174    270    165  591  609  232        389   \n",
       "3          295    276    274     83    263     81  519  557  259        282   \n",
       "4          307    293    268    161    275    150  575  575  289        386   \n",
       "...        ...    ...    ...    ...    ...    ...  ...  ...  ...        ...   \n",
       "6807425    393    379    266    308    297    266  321  269  296        382   \n",
       "6807426    232    273    217    285    265    275  510  363  277        562   \n",
       "6807427    271    263    265    210    264    203  104  612  304        500   \n",
       "6807428    280    271    264    170    263    166  237  225  251        283   \n",
       "6807429    253    261    249    142    259    141  118  612  241        453   \n",
       "\n",
       "         CD197  CD45RA    class  \n",
       "0          510     205  CD4_tcm  \n",
       "1          395     626  CD4_tcm  \n",
       "2          421     249  CD4_tcm  \n",
       "3          492     638  CD4_tcm  \n",
       "4          405     561  CD4_tcm  \n",
       "...        ...     ...      ...  \n",
       "6807425    359     333   debris  \n",
       "6807426    216     231   debris  \n",
       "6807427    263     804   debris  \n",
       "6807428    184     221   debris  \n",
       "6807429    227     374   debris  \n",
       "\n",
       "[6807430 rows x 13 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#col to drop CD107a, CD226, TIGIT,CD28\n",
    "total_grouped = total_grouped.drop(['CD107a', 'CD226',\"TIGIT\", \"CD28\", \"time\"], axis=1)\n",
    "total_grouped"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create the train and test data sets:\n",
    "train, test = train_test_split(total_grouped, test_size=0.2, random_state=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5445944, 13)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1361486, 13)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(6807430, 13)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "total_grouped.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>FSC_A</th>\n",
       "      <th>FSC_H</th>\n",
       "      <th>FSC_W</th>\n",
       "      <th>SSC_A</th>\n",
       "      <th>SSC_H</th>\n",
       "      <th>SSC_W</th>\n",
       "      <th>CD4</th>\n",
       "      <th>CD5</th>\n",
       "      <th>CD8</th>\n",
       "      <th>LIVE_DEAD</th>\n",
       "      <th>CD197</th>\n",
       "      <th>CD45RA</th>\n",
       "      <th>class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4392951</th>\n",
       "      <td>330</td>\n",
       "      <td>286</td>\n",
       "      <td>296</td>\n",
       "      <td>189</td>\n",
       "      <td>280</td>\n",
       "      <td>173</td>\n",
       "      <td>198</td>\n",
       "      <td>338</td>\n",
       "      <td>482</td>\n",
       "      <td>385</td>\n",
       "      <td>239</td>\n",
       "      <td>774</td>\n",
       "      <td>none_live</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5851456</th>\n",
       "      <td>270</td>\n",
       "      <td>264</td>\n",
       "      <td>262</td>\n",
       "      <td>205</td>\n",
       "      <td>267</td>\n",
       "      <td>197</td>\n",
       "      <td>626</td>\n",
       "      <td>242</td>\n",
       "      <td>269</td>\n",
       "      <td>617</td>\n",
       "      <td>212</td>\n",
       "      <td>687</td>\n",
       "      <td>debris</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1561533</th>\n",
       "      <td>357</td>\n",
       "      <td>292</td>\n",
       "      <td>313</td>\n",
       "      <td>154</td>\n",
       "      <td>277</td>\n",
       "      <td>143</td>\n",
       "      <td>589</td>\n",
       "      <td>560</td>\n",
       "      <td>260</td>\n",
       "      <td>356</td>\n",
       "      <td>486</td>\n",
       "      <td>788</td>\n",
       "      <td>CD4_th0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>554944</th>\n",
       "      <td>321</td>\n",
       "      <td>287</td>\n",
       "      <td>286</td>\n",
       "      <td>114</td>\n",
       "      <td>273</td>\n",
       "      <td>107</td>\n",
       "      <td>579</td>\n",
       "      <td>566</td>\n",
       "      <td>247</td>\n",
       "      <td>305</td>\n",
       "      <td>264</td>\n",
       "      <td>271</td>\n",
       "      <td>CD4_tem</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1786391</th>\n",
       "      <td>368</td>\n",
       "      <td>294</td>\n",
       "      <td>320</td>\n",
       "      <td>161</td>\n",
       "      <td>286</td>\n",
       "      <td>144</td>\n",
       "      <td>635</td>\n",
       "      <td>542</td>\n",
       "      <td>233</td>\n",
       "      <td>350</td>\n",
       "      <td>464</td>\n",
       "      <td>885</td>\n",
       "      <td>CD4_th0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         FSC_A  FSC_H  FSC_W  SSC_A  SSC_H  SSC_W  CD4  CD5  CD8  LIVE_DEAD  \\\n",
       "4392951    330    286    296    189    280    173  198  338  482        385   \n",
       "5851456    270    264    262    205    267    197  626  242  269        617   \n",
       "1561533    357    292    313    154    277    143  589  560  260        356   \n",
       "554944     321    287    286    114    273    107  579  566  247        305   \n",
       "1786391    368    294    320    161    286    144  635  542  233        350   \n",
       "\n",
       "         CD197  CD45RA      class  \n",
       "4392951    239     774  none_live  \n",
       "5851456    212     687     debris  \n",
       "1561533    486     788    CD4_th0  \n",
       "554944     264     271    CD4_tem  \n",
       "1786391    464     885    CD4_th0  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "train.to_csv(\"data/train_total.csv\", index = False)\n",
    "test.to_csv(\"data/test_total.csv\", index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}