IV. Final datasets.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "1e0f86b0",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "import random\n",
    "from sklearn.model_selection import train_test_split\n",
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "5098db5a",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import BertTokenizer\n",
    "tokenizer=BertTokenizer.from_pretrained('bert-base-uncased')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "3a856af5",
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(33)\n",
    "random.seed(33)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "686bb8cc",
   "metadata": {},
   "source": [
    "### Set up paths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "9aa631e8",
   "metadata": {},
   "outputs": [],
   "source": [
    "data='DATA'\n",
    "raw_data_folder='RAW_DATA'\n",
    "processed_data_folder='PROCESSED_DATA'\n",
    "word2vec_folder='word2vec'\n",
    "tfid_folder='tfid'\n",
    "final_dict='grp0_dict'\n",
    "train_folder='TRAIN'\n",
    "test_folder='TEST'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "8e975374",
   "metadata": {},
   "outputs": [],
   "source": [
    "if data not in os.listdir(os.getcwd()):\n",
    "    os.makedirs(os.path.join(data))\n",
    "DATA_PATH=os.path.join(data)\n",
    "if raw_data_folder not in os.listdir(os.path.join(DATA_PATH)):\n",
    "    os.makedirs(os.path.join(DATA_PATH,raw_data_folder))\n",
    "RAW_DATA_PATH=os.path.join(DATA_PATH,raw_data_folder)\n",
    "if processed_data_folder not in os.listdir(os.path.join(DATA_PATH)):\n",
    "    os.makedirs(os.path.join(DATA_PATH,processed_data_folder))\n",
    "PROCESSED_DATA_PATH=os.path.join(DATA_PATH,processed_data_folder)\n",
    "if raw_data_folder not in os.listdir(os.path.join(DATA_PATH)):\n",
    "    os.makedirs(os.path.join(DATA_PATH,raw_data_folder))\n",
    "if train_folder not in os.listdir(os.path.join(PROCESSED_DATA_PATH)):\n",
    "    os.makedirs(os.path.join(train_folder))\n",
    "TRAIN_PATH=os.path.join(PROCESSED_DATA_PATH,train_folder)\n",
    "if test_folder not in os.listdir(os.path.join(PROCESSED_DATA_PATH)):\n",
    "    os.makedirs(os.path.join(PROCESSED_DATA_PATH,test_folder))\n",
    "TEST_PATH=os.path.join(PROCESSED_DATA_PATH,test_folder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "20372d10",
   "metadata": {},
   "outputs": [],
   "source": [
    "df=pd.read_csv(os.path.join(RAW_DATA_PATH,'Labels.csv'), encoding = 'latin-1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "c1f6aba4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0.1</th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>Short description</th>\n",
       "      <th>Description</th>\n",
       "      <th>Caller</th>\n",
       "      <th>Assignment group</th>\n",
       "      <th>Lang_Short_Description</th>\n",
       "      <th>Lang_Description</th>\n",
       "      <th>Final_description</th>\n",
       "      <th>Total</th>\n",
       "      <th>Grp_change_1</th>\n",
       "      <th>group_cpy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>login issue</td>\n",
       "      <td>-verified user details.(employee# &amp; manager na...</td>\n",
       "      <td>spxjnwir pjlcoqds</td>\n",
       "      <td>GRP_0</td>\n",
       "      <td>en</td>\n",
       "      <td>en</td>\n",
       "      <td>login issue -verifie user details.(employee # ...</td>\n",
       "      <td>login issue -verifie user details.(employee # ...</td>\n",
       "      <td>5</td>\n",
       "      <td>GRP_0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>outlook</td>\n",
       "      <td>\\n\\nreceived from: hmjdrvpb.komuaywn@gmail.com...</td>\n",
       "      <td>hmjdrvpb komuaywn</td>\n",
       "      <td>GRP_0</td>\n",
       "      <td>en</td>\n",
       "      <td>en</td>\n",
       "      <td>outlook    receive from : hmjdrvpb.komuaywn@gm...</td>\n",
       "      <td>outlook    receive from : hmjdrvpb.komuaywn@gm...</td>\n",
       "      <td>1</td>\n",
       "      <td>GRP_0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>cant log in to vpn</td>\n",
       "      <td>\\n\\nreceived from: eylqgodm.ybqkwiam@gmail.com...</td>\n",
       "      <td>eylqgodm ybqkwiam</td>\n",
       "      <td>GRP_0</td>\n",
       "      <td>en</td>\n",
       "      <td>en</td>\n",
       "      <td>can not log in to vpn    receive from : eylqgo...</td>\n",
       "      <td>can not log in to vpn    receive from : eylqgo...</td>\n",
       "      <td>2</td>\n",
       "      <td>GRP_0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>unable to access hr_tool page</td>\n",
       "      <td>unable to access hr_tool page</td>\n",
       "      <td>xbkucsvz gcpydteq</td>\n",
       "      <td>GRP_0</td>\n",
       "      <td>en</td>\n",
       "      <td>en</td>\n",
       "      <td>unable to access hr_tool page xbkucsvz gcpydteq</td>\n",
       "      <td>unable to access hr_tool page</td>\n",
       "      <td>1</td>\n",
       "      <td>GRP_0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>skype error</td>\n",
       "      <td>skype error</td>\n",
       "      <td>owlgqjme qhcozdfx</td>\n",
       "      <td>GRP_0</td>\n",
       "      <td>en</td>\n",
       "      <td>en</td>\n",
       "      <td>skype error owlgqjme qhcozdfx</td>\n",
       "      <td>skype error</td>\n",
       "      <td>4</td>\n",
       "      <td>GRP_0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>unable to log in to engineering tool and skype</td>\n",
       "      <td>unable to log in to engineering tool and skype</td>\n",
       "      <td>eflahbxn ltdgrvkz</td>\n",
       "      <td>GRP_0</td>\n",
       "      <td>en</td>\n",
       "      <td>en</td>\n",
       "      <td>unable to log in to engineering tool and skype...</td>\n",
       "      <td>unable to log in to engineering tool and skype</td>\n",
       "      <td>2</td>\n",
       "      <td>GRP_0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>event: critical:HostName_221.company.com the v...</td>\n",
       "      <td>event: critical:HostName_221.company.com the v...</td>\n",
       "      <td>jyoqwxhz clhxsoqy</td>\n",
       "      <td>GRP_1</td>\n",
       "      <td>en</td>\n",
       "      <td>en</td>\n",
       "      <td>event : critical : hostname_221.company.com th...</td>\n",
       "      <td>event : critical : hostname_221.company.com th...</td>\n",
       "      <td>Other</td>\n",
       "      <td>GRP_1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>ticket_no1550391- employment status - new non-...</td>\n",
       "      <td>ticket_no1550391- employment status - new non-...</td>\n",
       "      <td>eqzibjhw ymebpoih</td>\n",
       "      <td>GRP_0</td>\n",
       "      <td>en</td>\n",
       "      <td>en</td>\n",
       "      <td>ticket_no1550391- employment status - new non ...</td>\n",
       "      <td>ticket_no1550391- employment status - new non ...</td>\n",
       "      <td>6</td>\n",
       "      <td>GRP_0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>unable to disable add ins on outlook</td>\n",
       "      <td>unable to disable add ins on outlook</td>\n",
       "      <td>mdbegvct dbvichlg</td>\n",
       "      <td>GRP_0</td>\n",
       "      <td>en</td>\n",
       "      <td>en</td>\n",
       "      <td>unable to disable add in on outlook mdbegvct d...</td>\n",
       "      <td>unable to disable add in on outlook</td>\n",
       "      <td>5</td>\n",
       "      <td>GRP_0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>ticket update on inplant_874773</td>\n",
       "      <td>ticket update on inplant_874773</td>\n",
       "      <td>fumkcsji sarmtlhy</td>\n",
       "      <td>GRP_0</td>\n",
       "      <td>en</td>\n",
       "      <td>en</td>\n",
       "      <td>ticket update on inplant_874773 fumkcsji sarmtlhy</td>\n",
       "      <td>ticket update on inplant_874773</td>\n",
       "      <td>0</td>\n",
       "      <td>GRP_0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0.1  Unnamed: 0  \\\n",
       "0             0           0   \n",
       "1             1           1   \n",
       "2             2           2   \n",
       "3             3           3   \n",
       "4             4           4   \n",
       "5             5           5   \n",
       "6             6           6   \n",
       "7             7           7   \n",
       "8             8           8   \n",
       "9             9           9   \n",
       "\n",
       "                                   Short description  \\\n",
       "0                                        login issue   \n",
       "1                                            outlook   \n",
       "2                                 cant log in to vpn   \n",
       "3                      unable to access hr_tool page   \n",
       "4                                        skype error   \n",
       "5     unable to log in to engineering tool and skype   \n",
       "6  event: critical:HostName_221.company.com the v...   \n",
       "7  ticket_no1550391- employment status - new non-...   \n",
       "8               unable to disable add ins on outlook   \n",
       "9                    ticket update on inplant_874773   \n",
       "\n",
       "                                         Description             Caller  \\\n",
       "0  -verified user details.(employee# & manager na...  spxjnwir pjlcoqds   \n",
       "1  \\n\\nreceived from: hmjdrvpb.komuaywn@gmail.com...  hmjdrvpb komuaywn   \n",
       "2  \\n\\nreceived from: eylqgodm.ybqkwiam@gmail.com...  eylqgodm ybqkwiam   \n",
       "3                      unable to access hr_tool page  xbkucsvz gcpydteq   \n",
       "4                                        skype error  owlgqjme qhcozdfx   \n",
       "5     unable to log in to engineering tool and skype  eflahbxn ltdgrvkz   \n",
       "6  event: critical:HostName_221.company.com the v...  jyoqwxhz clhxsoqy   \n",
       "7  ticket_no1550391- employment status - new non-...  eqzibjhw ymebpoih   \n",
       "8               unable to disable add ins on outlook  mdbegvct dbvichlg   \n",
       "9                    ticket update on inplant_874773  fumkcsji sarmtlhy   \n",
       "\n",
       "  Assignment group Lang_Short_Description Lang_Description  \\\n",
       "0            GRP_0                     en               en   \n",
       "1            GRP_0                     en               en   \n",
       "2            GRP_0                     en               en   \n",
       "3            GRP_0                     en               en   \n",
       "4            GRP_0                     en               en   \n",
       "5            GRP_0                     en               en   \n",
       "6            GRP_1                     en               en   \n",
       "7            GRP_0                     en               en   \n",
       "8            GRP_0                     en               en   \n",
       "9            GRP_0                     en               en   \n",
       "\n",
       "                                   Final_description  \\\n",
       "0  login issue -verifie user details.(employee # ...   \n",
       "1  outlook    receive from : hmjdrvpb.komuaywn@gm...   \n",
       "2  can not log in to vpn    receive from : eylqgo...   \n",
       "3    unable to access hr_tool page xbkucsvz gcpydteq   \n",
       "4                      skype error owlgqjme qhcozdfx   \n",
       "5  unable to log in to engineering tool and skype...   \n",
       "6  event : critical : hostname_221.company.com th...   \n",
       "7  ticket_no1550391- employment status - new non ...   \n",
       "8  unable to disable add in on outlook mdbegvct d...   \n",
       "9  ticket update on inplant_874773 fumkcsji sarmtlhy   \n",
       "\n",
       "                                               Total Grp_change_1 group_cpy  \n",
       "0  login issue -verifie user details.(employee # ...            5     GRP_0  \n",
       "1  outlook    receive from : hmjdrvpb.komuaywn@gm...            1     GRP_0  \n",
       "2  can not log in to vpn    receive from : eylqgo...            2     GRP_0  \n",
       "3                      unable to access hr_tool page            1     GRP_0  \n",
       "4                                        skype error            4     GRP_0  \n",
       "5     unable to log in to engineering tool and skype            2     GRP_0  \n",
       "6  event : critical : hostname_221.company.com th...        Other     GRP_1  \n",
       "7  ticket_no1550391- employment status - new non ...            6     GRP_0  \n",
       "8                unable to disable add in on outlook            5     GRP_0  \n",
       "9                    ticket update on inplant_874773            0     GRP_0  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a74841de",
   "metadata": {},
   "source": [
    "The features data will be got from final_description while the labels will be got from Grp_change_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "2aed9311",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0.1</th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>Short description</th>\n",
       "      <th>Description</th>\n",
       "      <th>Caller</th>\n",
       "      <th>Assignment group</th>\n",
       "      <th>Lang_Short_Description</th>\n",
       "      <th>Lang_Description</th>\n",
       "      <th>Final_description</th>\n",
       "      <th>Total</th>\n",
       "      <th>Grp_change_1</th>\n",
       "      <th>group_cpy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [Unnamed: 0.1, Unnamed: 0, Short description, Description, Caller, Assignment group, Lang_Short_Description, Lang_Description, Final_description, Total, Grp_change_1, group_cpy]\n",
       "Index: []"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df['Grp_change_1'].isna()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "9423f6b1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0.1</th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>Short description</th>\n",
       "      <th>Description</th>\n",
       "      <th>Caller</th>\n",
       "      <th>Assignment group</th>\n",
       "      <th>Lang_Short_Description</th>\n",
       "      <th>Lang_Description</th>\n",
       "      <th>Final_description</th>\n",
       "      <th>Total</th>\n",
       "      <th>Grp_change_1</th>\n",
       "      <th>group_cpy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [Unnamed: 0.1, Unnamed: 0, Short description, Description, Caller, Assignment group, Lang_Short_Description, Lang_Description, Final_description, Total, Grp_change_1, group_cpy]\n",
       "Index: []"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df['Final_description'].isna()]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a869edda",
   "metadata": {},
   "source": [
    "### Train test split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "286f27f8",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test=train_test_split(df['Final_description'].values,df['Grp_change_1'].values,test_size=0.1,stratify=df['Grp_change_1'],random_state=33)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e7812e60",
   "metadata": {},
   "source": [
    "### Tokenizing the features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "a3657c4c",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_features=tokenizer(list(X_train),add_special_tokens=True,max_length=512,padding='max_length',truncation=True,return_tensors='pt')\n",
    "test_features=tokenizer(list(X_test),add_special_tokens=True,max_length=512,padding='max_length',truncation=True,return_tensors='pt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "58fa308c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([7650, 512])"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_features['input_ids'].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "9fd9be54",
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(train_features,os.path.join(TRAIN_PATH,'train_features.pt'))\n",
    "torch.save(test_features,os.path.join(TEST_PATH,'test_features.pt'))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "72795e49",
   "metadata": {},
   "source": [
    "### Creating the labels "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "6066542e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "28"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df['Grp_change_1'].unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "6c305d63",
   "metadata": {},
   "outputs": [],
   "source": [
    "dict_val_to_target={}\n",
    "dict_target_to_val={}\n",
    "for key, value in enumerate(df['Grp_change_1'].unique()):\n",
    "    dict_val_to_target[value]=key\n",
    "for key, value in dict_val_to_target.items():\n",
    "    dict_target_to_val[value]=key"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "7814a40d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'5': 0, '1': 1, '2': 2, '4': 3, 'Other': 4, '6': 5, '0': 6, 'GRP_3': 7, '3': 8, '8': 9, 'GRP_4': 10, 'GRP_5': 11, 'GRP_6': 12, 'GRP_8': 13, 'GRP_9': 14, 'GRP_10': 15, 'GRP_12': 16, 'GRP_13': 17, 'GRP_14': 18, '7': 19, 'GRP_16': 20, 'GRP_18': 21, 'GRP_19': 22, 'GRP_2': 23, 'GRP_24': 24, 'GRP_25': 25, 'GRP_29': 26, 'GRP_33': 27} \n",
      "\n",
      "{0: '5', 1: '1', 2: '2', 3: '4', 4: 'Other', 5: '6', 6: '0', 7: 'GRP_3', 8: '3', 9: '8', 10: 'GRP_4', 11: 'GRP_5', 12: 'GRP_6', 13: 'GRP_8', 14: 'GRP_9', 15: 'GRP_10', 16: 'GRP_12', 17: 'GRP_13', 18: 'GRP_14', 19: '7', 20: 'GRP_16', 21: 'GRP_18', 22: 'GRP_19', 23: 'GRP_2', 24: 'GRP_24', 25: 'GRP_25', 26: 'GRP_29', 27: 'GRP_33'} \n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(dict_val_to_target,'\\n')\n",
    "print(dict_target_to_val,'\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "659065cb",
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train_=torch.tensor([dict_val_to_target[y_tr] for y_tr in y_train])\n",
    "y_test_=torch.tensor([dict_val_to_target[y_ts] for y_ts in y_test])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "35e1a09a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([7650])"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train_.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "aa948e84",
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(y_train_,os.path.join(TRAIN_PATH,'train_labels.pt'))\n",
    "torch.save(y_test_,os.path.join(TEST_PATH,'test_labels.pt'))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8e8018b2",
   "metadata": {},
   "source": [
    "### Getting the weights balance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "9784ce7a",
   "metadata": {},
   "outputs": [],
   "source": [
    "y=np.array([dict_val_to_target[y_tr] for y_tr in df['Grp_change_1']])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "dbadd307",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.57823129, 0.29022125, 0.65993789, 0.40314931, 0.27597403,\n",
       "       0.90080543, 1.99718045, 1.51785714, 0.83170254, 1.53318903,\n",
       "       3.03571429, 2.35326689, 1.64984472, 0.45926086, 1.20464853,\n",
       "       2.16836735, 1.18121178, 2.09359606, 2.57263923, 2.16836735,\n",
       "       3.57142857, 3.44967532, 1.41196013, 1.25963248, 1.05042017,\n",
       "       2.61699507, 3.12960236, 2.83711615])"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.utils.class_weight import compute_class_weight\n",
    "balance=compute_class_weight(class_weight='balanced', classes=np.unique(y), y=y)\n",
    "balance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "e0052e20",
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save(os.path.join(TRAIN_PATH,'balance.npy'),balance)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "95cc80ed",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "28"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(balance)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}