tf training.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "59c498e6",
   "metadata": {},
   "outputs": [],
   "source": [
    "%config Completer.use_jedi=False\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "490d329d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-05-03 23:43:50.997867: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory\n",
      "2022-05-03 23:43:50.997895: I tensorflow/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.\n",
      "/tmp/ipykernel_4911/982806629.py:5: FutureWarning: Passing a negative integer is deprecated in version 1.0 and will not be supported in future version. Instead, use None to not limit the column width.\n",
      "  pd.set_option(\"display.max_colwidth\", -1)\n"
     ]
    }
   ],
   "source": [
    "from datasets import load_dataset\n",
    "from labels import mapping\n",
    "\n",
    "import pandas as pd\n",
    "pd.set_option(\"display.max_colwidth\", -1)\n",
    "import numpy as np\n",
    "\n",
    "from matplotlib import pyplot as plt\n",
    "from tqdm.notebook import tqdm\n",
    "\n",
    "from torch.utils.data import DataLoader\n",
    "\n",
    "import tez\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import transformers\n",
    "from sklearn import metrics, model_selection, preprocessing\n",
    "from transformers import AdamW, get_linear_schedule_with_warmup"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "49b084fd",
   "metadata": {},
   "source": [
    "Load the dataset and split it into train, valid, and test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "3d32395e",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No config specified, defaulting to: go_emotions/simplified\n",
      "Reusing dataset go_emotions (/home/calixto/.cache/huggingface/datasets/go_emotions/simplified/0.0.0/2637cfdd4e64d30249c3ed2150fa2b9d279766bfcd6a809b9f085c61a90d776d)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4b982e8ae0ac48d4baecc10f4c26562e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "go_emotions = load_dataset(\"go_emotions\")\n",
    "\n",
    "data = go_emotions.data\n",
    "\n",
    "train = go_emotions.data[\"train\"].to_pandas()\n",
    "valid = go_emotions.data[\"validation\"].to_pandas()\n",
    "test = go_emotions.data[\"test\"].to_pandas()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "198d76c0",
   "metadata": {},
   "source": [
    "Convert the dataset to a one-hot representation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "fd7b116a",
   "metadata": {},
   "outputs": [],
   "source": [
    "n_labels = len(mapping)\n",
    "\n",
    "def one_hot_labels(df):\n",
    "    dict_labels = []\n",
    "    for i in tqdm(range(len(df)), leave=False):\n",
    "        d = dict(zip(range(n_labels), [0]*n_labels))\n",
    "        labels = df.loc[i][\"labels\"]\n",
    "        for label in labels:\n",
    "            d[label] = 1\n",
    "        dict_labels.append(d)\n",
    "    df_labels = pd.DataFrame(dict_labels)\n",
    "    return df_labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "f410ae4f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/43410 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/5426 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/5427 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "train_oh_labels = one_hot_labels(train)\n",
    "valid_oh_labels = one_hot_labels(valid)\n",
    "test_oh_labels = one_hot_labels(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "c5814ddf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(43410, 28)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_oh_labels.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "3a8c2f7f",
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.concat([train, train_oh_labels], axis=1)\n",
    "valid = pd.concat([valid, valid_oh_labels], axis=1)\n",
    "test = pd.concat([test, test_oh_labels], axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fbea0168",
   "metadata": {},
   "source": [
    "Quick check of the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "a27b7704",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(43410, 31) (5426, 31) (5427, 31)\n"
     ]
    }
   ],
   "source": [
    "print(train.shape, valid.shape, test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "4708947c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>labels</th>\n",
       "      <th>id</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>...</th>\n",
       "      <th>18</th>\n",
       "      <th>19</th>\n",
       "      <th>20</th>\n",
       "      <th>21</th>\n",
       "      <th>22</th>\n",
       "      <th>23</th>\n",
       "      <th>24</th>\n",
       "      <th>25</th>\n",
       "      <th>26</th>\n",
       "      <th>27</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>My favourite food is anything I didn't have to cook myself.</td>\n",
       "      <td>[27]</td>\n",
       "      <td>eebbqej</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Now if he does off himself, everyone will think hes having a laugh screwing with people instead of actually dead</td>\n",
       "      <td>[27]</td>\n",
       "      <td>ed00q6i</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>WHY THE FUCK IS BAYLESS ISOING</td>\n",
       "      <td>[2]</td>\n",
       "      <td>eezlygj</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>To make her feel threatened</td>\n",
       "      <td>[14]</td>\n",
       "      <td>ed7ypvh</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Dirty Southern Wankers</td>\n",
       "      <td>[3]</td>\n",
       "      <td>ed0bdzj</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 31 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                               text  \\\n",
       "0  My favourite food is anything I didn't have to cook myself.                                                        \n",
       "1  Now if he does off himself, everyone will think hes having a laugh screwing with people instead of actually dead   \n",
       "2  WHY THE FUCK IS BAYLESS ISOING                                                                                     \n",
       "3  To make her feel threatened                                                                                        \n",
       "4  Dirty Southern Wankers                                                                                             \n",
       "\n",
       "  labels       id  0  1  2  3  4  5  6  ...  18  19  20  21  22  23  24  25  \\\n",
       "0  [27]   eebbqej  0  0  0  0  0  0  0  ...  0   0   0   0   0   0   0   0    \n",
       "1  [27]   ed00q6i  0  0  0  0  0  0  0  ...  0   0   0   0   0   0   0   0    \n",
       "2  [2]    eezlygj  0  0  1  0  0  0  0  ...  0   0   0   0   0   0   0   0    \n",
       "3  [14]   ed7ypvh  0  0  0  0  0  0  0  ...  0   0   0   0   0   0   0   0    \n",
       "4  [3]    ed0bdzj  0  0  0  1  0  0  0  ...  0   0   0   0   0   0   0   0    \n",
       "\n",
       "   26  27  \n",
       "0  0   1   \n",
       "1  0   1   \n",
       "2  0   0   \n",
       "3  0   0   \n",
       "4  0   0   \n",
       "\n",
       "[5 rows x 31 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "a76ecd93",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:title={'center':'distribution of labels in test'}>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA3AAAAL0CAYAAABNvNWtAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAABmRElEQVR4nO3de5hdZX33//fHIHhAkEM8cAxgULEqaASt9fBTwCCWWEWJtgpqS62iVh9bsfqIxaqoT7VWsZVW1FoxIJ5iiQJWtJ7AREQxIBKQQ+IpnKQKooHv74+1Rne2M8yeZFYmK/N+Xde+svY6fe+1Z2dmffa6171TVUiSJEmSNn93mekGSJIkSZJGY4CTJEmSpJ4wwEmSJElSTxjgJEmSJKknDHCSJEmS1BMGOEmSJEnqCQOcJG2BknwoyT+0049Lctk07vtzSY5up49J8tVp3PefJjlnuvY3hbqPTXJ5kl8kefo4y69KcvCI+6okD9jAdkx525l6zQbqT+v7S5J05wxwkrSFq6qvVNUDJ1svyRuT/OcI+zusqj68se1KMq8NLFsN7PujVXXoxu57A5wIvLeqtq2qT89A/Q22Ma/ZqD/zSeqP9P6SJE0PA5wkaSRpbKl/N/YEVs50IzY3W/jPXJJ6yV/KkrQFSHJAkguT/G+S04G7DSx7YpLVA89fk2RNu+5lSZ6cZCHwd8BRbTfC77TrfinJm5N8DbgF2Lud9+frl897k/w8yfeTPHlgwXpdD4eu+PxP++9Nbc3HDHfJTPKHSZa3+16e5A8Hln0pyZuSfK09lnOS7Hwnr9FfJFmV5IYkS5Ps0s6/Atgb+Gzbjm0mea0PTPKNJDcl+XF77FsPrfbUJFcmuS7JOwZDUJIXJrk0yY1Jzk6y5wR1nprkkvbY1iR59QTrDb9mleTFbZfQm5KcnCTjbDeVn/kL2jb/b3tcfzmwn+H311VJXp3ku+3P7fQkdxuuL0naMAY4Seq5Njx8GvgIsCPwceCZE6z7QOA44FFVdS/gKcBVVfV54C3A6W03wocPbPY84FjgXsDV4+z2IOAKYGfgBOCTSXYcoemPb/+9d1vzG0Nt3RE4C/hnYCfgncBZSXYaWO25wAuA+wBbAxOFnCcBbwWeDdy/PY4lAFW1D3AN8MdtO26bpN23A69sj/cxwJOBlwyt8yfAAuARwCLghW07FtGEpmcAc4GvAB+boM4HgL9sf05/AHxxknYNehrwKOBhNMf8lOEVpvgz/1m7z+1oXu93JXnEndR/NrAQ2KttwzFTaLsk6U4Y4CSp/x4N3BX4p6r6TVWdCSyfYN3bgW2A/ZLctaquqqorJtn/h6pqZVWtq6rfjLP8ZwO1TwcuAw7fwGMZdDhweVV9pK39MeD7wB8PrPPBqvpBVd0KnAHsP8G+/hQ4taoubAPaa4HHJJk31UZV1beq6vy2TVcB7weeMLTa26rqhqq6Bvgn4Dnt/BcDb62qS6tqHU2A2n+Cq3C/ofk5bVdVN1bVhVNo5klVdVNb/zwmfl0mst7PvKrOqqorqvFl4BzgcXey/T9X1Y+q6gbgsxtQX5I0AQOcJPXfLsCaqqqBeeNdKaOqVgF/DbwR+FmSJWNdCe/EtZMsH6/2ZPscxS78/nFcDew68PwnA9O3ANuOsq+q+gVw/dC+RpJk3yT/leQnSW6mCWHDXTcHX7PB12NP4N1t18abgBuATNCOZwJPBa5O8uUkj5lCM0d9XSay3s88yWFJzm+7n97UtmvC7qrTUF+SNAEDnCT134+BXYfuc9pjopWr6rSq+iOaMFHA28YWTbTJJPXHq/2jdvqXwD0Glt1vCvv9UdvGQXsAaybZbtJ9JbknTbfMDdnXv9BcCZxfVdvRdIkcvsds94HpwdfjWppukfceeNy9qr4+XKSqllfVIpruoZ+mucI43Sb9mbf3BH4C+H/Afavq3sAyfv+YJUmbgAFOkvrvG8A64OVJ7prkGcCB462Y5IFJntSelP8KuBW4o138U2Bepj7q4H0Gaj8LeDDNCT7ARcDidtkC4MiB7da2tfeeYL/LgH2TPDfJVkmOAvYD/muK7YPmPrMXJNm/Pfa3ABe0XSCn6l7AzcAvkjwI+Ktx1vmbJDsk2R14BXB6O/9fgdcmeQhAku3b12w9SbZO8/1u27fdVm/mdz+n6TTKz3xrmm63a4F1SQ4DZuKrHiRJGOAkqfeq6tc0g2IcQ9Ml7yjgkxOsvg1wEnAdTTe3+9DcDwbN4CcA1yeZyv1WFwDz232+GTiyqq5vl/1fYB/gRuDvgdMG2n1Lu/7X2i6Fjx46rutpBs74PzTdHf8WeFpVXTeFto3t6wttWz5Bc8VyH2DxVPfTejXN4Cn/C/wbvwtngz4DfIsmwJ5FMyAJVfUpmiueS9rul98DDpugzvOAq9r1XkxzH990m/RnXlX/C7yc5grgjTTHvrSDtkiSRpD1b1uQJEmSJG2uvAInSZIkST1hgJMkSZKknjDASZIkSVJPGOAkSZIkqSe2mukGDNt5551r3rx5M90MSZIkSZoR3/rWt66rqrnjLdvsAty8efNYsWLFTDdDkiRJkmZEkqsnWmYXSkmSJEnqCQOcJEmSJPWEAU6SJEmSesIAJ0mSJEk9YYCTJEmSpJ4wwEmSJElSTxjgJEmSJKknDHCSJEmS1BOb3Rd5S5IkSVIfzDv+rA3e9qqTDt+g7bwCJ0mSJEk9YYCTJEmSpJ4wwEmSJElSTxjgJEmSJKknDHCSJEmS1BMGOEmSJEnqCQOcJEmSJPWEAU6SJEmSesIAJ0mSJEk9YYCTJEmSpJ4wwEmSJElSTxjgJEmSJKknDHCSJEmS1BMjBbgkC5NclmRVkuPHWf7iJBcnuSjJV5PsN7Dste12lyV5ynQ2XpIkSZJmk0kDXJI5wMnAYcB+wHMGA1rrtKp6aFXtD7wdeGe77X7AYuAhwELgfe3+JEmSJElTNMoVuAOBVVV1ZVX9GlgCLBpcoapuHnh6T6Da6UXAkqq6rap+CKxq9ydJkiRJmqKtRlhnV+DageergYOGV0ryUuBVwNbAkwa2PX9o213H2fZY4FiAPfbYY5R2S5IkSdKsM22DmFTVyVW1D/Aa4PVT3PaUqlpQVQvmzp07XU2SJEmSpC3KKAFuDbD7wPPd2nkTWQI8fQO3lSRJkiRNYJQAtxyYn2SvJFvTDEqydHCFJPMHnh4OXN5OLwUWJ9kmyV7AfOCbG99sSZIkSZp9Jr0HrqrWJTkOOBuYA5xaVSuTnAisqKqlwHFJDgZ+A9wIHN1uuzLJGcAlwDrgpVV1e0fHIkmSJElbtFEGMaGqlgHLhua9YWD6FXey7ZuBN29oAyVJkiRJjWkbxESSJEmS1C0DnCRJkiT1hAFOkiRJknrCACdJkiRJPWGAkyRJkqSeMMBJkiRJUk8Y4CRJkiSpJwxwkiRJktQTBjhJkiRJ6gkDnCRJkiT1hAFOkiRJknrCACdJkiRJPWGAkyRJkqSeMMBJkiRJUk8Y4CRJkiSpJwxwkiRJktQTBjhJkiRJ6gkDnCRJkiT1hAFOkiRJknrCACdJkiRJPWGAkyRJkqSeGCnAJVmY5LIkq5IcP87yVyW5JMl3k/x3kj0Hlt2e5KL2sXQ6Gy9JkiRJs8lWk62QZA5wMnAIsBpYnmRpVV0ysNq3gQVVdUuSvwLeDhzVLru1qvaf3mZLkiRJ0uwzyhW4A4FVVXVlVf0aWAIsGlyhqs6rqlvap+cDu01vMyVJkiRJowS4XYFrB56vbudN5EXA5wae3y3JiiTnJ3n6eBskObZdZ8XatWtHaJIkSZIkzT6TdqGciiR/BiwAnjAwe8+qWpNkb+CLSS6uqisGt6uqU4BTABYsWFDT2SZJkiRJ2lKMcgVuDbD7wPPd2nnrSXIw8DrgiKq6bWx+Va1p/70S+BJwwEa0V5IkSZJmrVEC3HJgfpK9kmwNLAbWG00yyQHA+2nC288G5u+QZJt2emfgscDg4CeSJEmSpBFN2oWyqtYlOQ44G5gDnFpVK5OcCKyoqqXAO4BtgY8nAbimqo4AHgy8P8kdNGHxpKHRKyVJkiRJIxrpHriqWgYsG5r3hoHpgyfY7uvAQzemgZIkSZKkxkhf5C1JkiRJmnkGOEmSJEnqCQOcJEmSJPWEAU6SJEmSesIAJ0mSJEk9YYCTJEmSpJ4wwEmSJElSTxjgJEmSJKknDHCSJEmS1BMGOEmSJEnqCQOcJEmSJPWEAU6SJEmSesIAJ0mSJEk9YYCTJEmSpJ4wwEmSJElSTxjgJEmSJKknDHCSJEmS1BMGOEmSJEnqCQOcJEmSJPWEAU6SJEmSesIAJ0mSJEk9MVKAS7IwyWVJViU5fpzlr0pySZLvJvnvJHsOLDs6yeXt4+jpbLwkSZIkzSaTBrgkc4CTgcOA/YDnJNlvaLVvAwuq6mHAmcDb2213BE4ADgIOBE5IssP0NV+SJEmSZo9RrsAdCKyqqiur6tfAEmDR4ApVdV5V3dI+PR/YrZ1+CnBuVd1QVTcC5wILp6fpkiRJkjS7jBLgdgWuHXi+up03kRcBn5vKtkmOTbIiyYq1a9eO0CRJkiRJmn2mdRCTJH8GLADeMZXtquqUqlpQVQvmzp07nU2SJEmSpC3GKAFuDbD7wPPd2nnrSXIw8DrgiKq6bSrbSpIkSZImN0qAWw7MT7JXkq2BxcDSwRWSHAC8nya8/Wxg0dnAoUl2aAcvObSdJ0mSJEmaoq0mW6Gq1iU5jiZ4zQFOraqVSU4EVlTVUpouk9sCH08CcE1VHVFVNyR5E00IBDixqm7o5EgkSZIkaQs3aYADqKplwLKheW8YmD74TrY9FTh1QxsoSZIkSWpM6yAmkiRJkqTuGOAkSZIkqScMcJIkSZLUEwY4SZIkSeoJA5wkSZIk9YQBTpIkSZJ6wgAnSZIkST1hgJMkSZKknjDASZIkSVJPGOAkSZIkqScMcJIkSZLUEwY4SZIkSeoJA5wkSZIk9YQBTpIkSZJ6wgAnSZIkST1hgJMkSZKknjDASZIkSVJPGOAkSZIkqScMcJIkSZLUEwY4SZIkSeoJA5wkSZIk9cRIAS7JwiSXJVmV5Phxlj8+yYVJ1iU5cmjZ7Ukuah9Lp6vhkiRJkjTbbDXZCknmACcDhwCrgeVJllbVJQOrXQMcA7x6nF3cWlX7b3xTJUmSJGl2mzTAAQcCq6rqSoAkS4BFwG8DXFVd1S67o4M2SpIkSZIYrQvlrsC1A89Xt/NGdbckK5Kcn+TpU2mcJEmSJOl3RrkCt7H2rKo1SfYGvpjk4qq6YnCFJMcCxwLssccem6BJkiRJktQ/o1yBWwPsPvB8t3beSKpqTfvvlcCXgAPGWeeUqlpQVQvmzp076q4lSZIkaVYZJcAtB+Yn2SvJ1sBiYKTRJJPskGSbdnpn4LEM3DsnSZIkSRrdpAGuqtYBxwFnA5cCZ1TVyiQnJjkCIMmjkqwGngW8P8nKdvMHAyuSfAc4DzhpaPRKSZIkSdKIRroHrqqWAcuG5r1hYHo5TdfK4e2+Djx0I9soSZIkSWLEL/KWJEmSJM08A5wkSZIk9YQBTpIkSZJ6wgAnSZIkST1hgJMkSZKknjDASZIkSVJPGOAkSZIkqScMcJIkSZLUEwY4SZIkSeoJA5wkSZIk9YQBTpIkSZJ6wgAnSZIkST1hgJMkSZKknjDASZIkSVJPGOAkSZIkqScMcJIkSZLUEwY4SZIkSeoJA5wkSZIk9YQBTpIkSZJ6wgAnSZIkST1hgJMkSZKknjDASZIkSVJPjBTgkixMclmSVUmOH2f545NcmGRdkiOHlh2d5PL2cfR0NVySJEmSZptJA1ySOcDJwGHAfsBzkuw3tNo1wDHAaUPb7gicABwEHAickGSHjW+2JEmSJM0+o1yBOxBYVVVXVtWvgSXAosEVquqqqvoucMfQtk8Bzq2qG6rqRuBcYOE0tFuSJEmSZp1RAtyuwLUDz1e380Yx0rZJjk2yIsmKtWvXjrhrSZIkSZpdNotBTKrqlKpaUFUL5s6dO9PNkSRJkqTN0igBbg2w+8Dz3dp5o9iYbSVJkiRJA7YaYZ3lwPwke9GEr8XAc0fc/9nAWwYGLjkUeO2UWzlg3vFnbfC2V510+MaUliRJkqQZNekVuKpaBxxHE8YuBc6oqpVJTkxyBECSRyVZDTwLeH+Sle22NwBvogmBy4ET23mSJEmSpCka5QocVbUMWDY07w0D08tpukeOt+2pwKkb0UZJkiRJEpvJICaSJEmSpMkZ4CRJkiSpJwxwkiRJktQTBjhJkiRJ6gkDnCRJkiT1hAFOkiRJknrCACdJkiRJPWGAkyRJkqSeMMBJkiRJUk8Y4CRJkiSpJwxwkiRJktQTBjhJkiRJ6gkDnCRJkiT1xFYz3QBpzLzjz9rgba866fBpbIkkSZK0efIKnCRJkiT1hAFOkiRJknrCACdJkiRJPeE9cJIk9YD3CUuSwCtwkiRJktQbBjhJkiRJ6gm7UGpcdtWRJEmSNj8jXYFLsjDJZUlWJTl+nOXbJDm9XX5Bknnt/HlJbk1yUfv412luvyRJkiTNGpNegUsyBzgZOARYDSxPsrSqLhlY7UXAjVX1gCSLgbcBR7XLrqiq/ae32ZueV6QkSZIkzbRRrsAdCKyqqiur6tfAEmDR0DqLgA+302cCT06S6WumJEmSJGmUe+B2Ba4deL4aOGiidapqXZKfAzu1y/ZK8m3gZuD1VfWV4QJJjgWOBdhjjz2mdACSpsaryZIkSf3V9SAmPwb2qKrrkzwS+HSSh1TVzYMrVdUpwCkACxYsqI7bJEmSNC4/5JK0uRslwK0Bdh94vls7b7x1VifZCtgeuL6qCrgNoKq+leQKYF9gxcY2XJI2Z54ESpKkLoxyD9xyYH6SvZJsDSwGlg6tsxQ4up0+EvhiVVWSue0gKCTZG5gPXDk9TZckSZKk2WXSK3DtPW3HAWcDc4BTq2plkhOBFVW1FPgA8JEkq4AbaEIewOOBE5P8BrgDeHFV3dDFgUiSJEnSlm6ke+CqahmwbGjeGwamfwU8a5ztPgF8YiPbKEmSJElixC/yliRJkiTNPAOcJEmSJPVE118joI3kSHaSJKkrnmdI/eMVOEmSJEnqCa/ASZKkzY5XhiRpfAY4aYZ4ciJJkqSpMsBJkiRJmhZ+QN09A5ykLZp/SCRJ0pbEQUwkSZIkqSe8AqdZzys0kiTNDv7N33LNpp+tV+AkSZIkqScMcJIkSZLUE3ahlCT11oZ2melbdxlJksYY4CRpCzKb7gGQJGk2sgulJEmSJPWEV+AkbRJeGZIkSdp4BjhJkqbADyMkSTPJACdJkiRtYfywacvlPXCSJEmS1BMGOEmSJEnqCbtQSpI2ml11JEnaNEa6ApdkYZLLkqxKcvw4y7dJcnq7/IIk8waWvbadf1mSp0xj2yVJkiRpVpn0ClySOcDJwCHAamB5kqVVdcnAai8CbqyqByRZDLwNOCrJfsBi4CHALsAXkuxbVbdP94FIkiT1lVext1z+bDXdRulCeSCwqqquBEiyBFgEDAa4RcAb2+kzgfcmSTt/SVXdBvwwyap2f9+YnuZLkqQuefKpLvi+kjZcqurOV0iOBBZW1Z+3z58HHFRVxw2s8712ndXt8yuAg2hC3flV9Z/t/A8An6uqM4dqHAsc2z59IHDZBh7PzsB1G7jtxrCuda3bz7qz6Vita13r9rOmda1r3dlZd8+qmjvegs1iEJOqOgU4ZWP3k2RFVS2YhiZZ17rWnQV1Z9OxWte61u1nTeta17rWHTbKICZrgN0Hnu/Wzht3nSRbAdsD14+4rSRJkiRpBKMEuOXA/CR7JdmaZlCSpUPrLAWObqePBL5YTd/MpcDidpTKvYD5wDenp+mSJEmSNLtM2oWyqtYlOQ44G5gDnFpVK5OcCKyoqqXAB4CPtIOU3EAT8mjXO4NmwJN1wEs7HoFyo7thWte61p1VdWfTsVrXutbtZ03rWte61l3PpIOYSJIkSZI2DyN9kbckSZIkaeYZ4CRJkiSpJwxwkiRJktQTm8X3wG2oJA8CFgG7trPWAEur6tKZa1V32uPdFbigqn4xMH9hVX2+w7oHAlVVy5PsBywEvl9Vy7qqOU4b/qOqnr+p6g3U/SPgQOB7VXVOh3UOAi6tqpuT3B04HngEzQBAb6mqn3dQ8+XAp6rq2une9yR1x0az/VFVfSHJc4E/BC4FTqmq33RYe2/gGTRfb3I78APgtKq6uauakiRJ06m3g5gkeQ3wHGAJsLqdvRvNieGSqjppBtr0gqr6YEf7fjnwUpqT3P2BV1TVZ9plF1bVIzqqewJwGE3YPxc4CDgPOAQ4u6re3EHN4a+pCPD/AV8EqKojprvmQO1vVtWB7fRf0LzmnwIOBT7b1fsqyUrg4e2or6cAtwBnAk9u5z+jg5o/B34JXAF8DPh4Va2d7jrj1P0ozfvpHsBNwLbAJ2mONVV19MRbb1TdlwNPA/4HeCrw7bb+nwAvqaovdVFXkqTNUZL7VNXPZrodm0KSnarq+plux3Tpc4D7AfCQ4U/r20/3V1bV/Blo0zVVtUdH+74YeExV/SLJPJqT+49U1buTfLuqDuiw7v7ANsBPgN0GrhJdUFUP66DmhTRXnv4dKJoA9zF+9/UUX57umgO1f/taJlkOPLWq1ia5J3B+VT20o7qXVtWD2+n1AnmSi6pq/w5qfht4JHAwcBRwBPAtmtf6k1X1v9Nds6373ap6WJKtaK6a71JVtycJ8J0u3lNt3YuB/dta9wCWVdUTk+wBfKar/0OafWbTSRFseSdGs1WS7YHXAk8H7kPz9/dnwGeAk6rqphlo0+eq6rCO9r0dzfHuBnyuqk4bWPa+qnpJR3XvB5wA3AG8AXgZ8EyaD+hfUVU/7qjujsOzaP7mH0CTB27ooOZve4i17693Ao8Cvge8sqp+Ot0121onAf+vqq5LsgA4g+b1vivw/K7OI9vz108CH6uqK7qoMabP98DdAewyzvz7t8s6keS7EzwuBu7bVV3gLmPdJqvqKuCJwGFJ3knzn7Ar66rq9qq6BbhirKtZVd1Kd6/zAppfKq8Dft5eGbm1qr7cZXhr3SXJDkl2ovmFthagqn5J812GXflekhe0099pf+GQZF+gqy6FVVV3VNU5VfUimv9P76PpIntlRzWheY23Bu5FcxVu+3b+NjS/XLs01m18G5orf1TVNV3WTbJ9kpOSfD/JDUmuT3JpO+/eXdWdpE2f63Df2yV5a5KPtN1jB5e9r8O690vyL0lOTrJTkjcmuTjJGUnu32HdHYceOwHfbH+PDJ8wTWfdhQPT2yf5QPu36LQknf0tat+3O7fTC5JcCVyQ5OokT+iw7oVJXp9kn65qTFB3QZLzkvxnkt2TnJvk50mWJ+nqg9Ntk5yYZGVba22S85Mc00W9AWcANwJPrKodq2onmt4vN7bLOpHkERM8HknzAXJXPkhz/vQJYHGSTyTZpl326A7rfojmQ+praXo03UrTK+QrwL92WPc6mnOrsccKmttyLmynu/CWgel/BH4M/DGwHHh/RzUBDq+q69rpdwBHVdUDaHqP/WOHdXcA7g2cl+SbSV6ZZLysstH6fA/cXwP/neRymv8EAHsADwCO67DufYGn0PxCGxTg6x3W/WmS/avqIoD2StzTgFOBTq4KtX6d5B5tgHvk2Mz2k5ROAlxV3QG8K8nH239/yqZ7r25P84stQCW5f1X9OMm2dBuU/xx4d5LX0/yS/UaSa2ne23/eUc31jqe9mr0UWNpeoerKB4DvA3NoQvrH25PAR9N0ie7KvwPLk1wAPA54G0CSucC0f/I44Aya7r9PrKqftDXvBxzdLju0i6JJJupWHbo/Kbqc5qTohUmeCTy3qm6j+5Ois4B70pwUfZTmpOjpNCdFizqqex1w9dC8sZOiAvbuqO5bgLF7nwdPjJ5Bc2L09I7qHl5Vx7fTYydGy9sPm06j+QCuC4MnRj+h6SlwelX9qKN6Y95Hc7Xk3jR/419ZVYckeXK77DEd1PwoTdf9pwDPpnlPLwFen2Tfqvq7DmoCzKuqtw3OaH9nvS3JCzuqCc3J/JcZ/2/svTusu09VPbOd/nSS1wFfTNLZbRqt+1bVewCSvGTgNX9Pkhd1WPdvaALM31TVxW39H1bVXh3WHLRgoDfRu5J0crtEa6skW1XVOuDuVbUcoKp+MBDSu3BjVb0aeHWSx9Hc6nVhkktprspN2xd697YLJUCSu9AMMDE4iMnyqrq9w5ofAD5YVV8dZ9lpVfXccTabjrq70VwN+8k4yx5bVV/rqO427YnX8PydgfuP/RLoUpLDgcd2+EdrlDbcg+aX7g87rrMdsBdNYF3dVfeCtta+VfWDrvY/Se1dAKrqR+1VqIOBa6rqmx3XfQjwYJpBab7fZa2BmpdV1QOnumwa6t7OxCdFj66qu3dUd70uv+1J0VNpuuie2+H9uoPdn9frzj7cpmmu+3+YgZOiDHS1Huc17/J4LwUe2t6ve35VPXpg2cUddjMfPN6xE6Nn0HQ7m9YTo6G6d/a+6uT2hSTfqaqHDzxfXlWPas95LqmqB013zbbOOcAXgA+P/e1pr+YeAxxSVQd3VPd7wJ9U1eXjLLu2qnbvqO6lNLfi3DEw7xiaoLNtVe3ZUd3f/nyT/ENVvX5gWWf/h9r97wa8i+bD4RNoblvo6kMmkqym6TYZmjEF9qk2eKS9naKjui+j+UDrJODxNB8AfRJ4ErB3VT2vo7q/NyZFkjk0fyOOqqoXjL/l1PX5CtzYlZrzN3HNCT8d6Sq8tftefSfLOglv7b5/L7y186+j+eS5c1V1Fs0n6zOmvQLZaXhr69wMfKfrOm2tGQlvbe0fDUzfRHNP56aouxJYuSlqDbg6yd8y/klRlyOAXgr85UQnRR3W3SbJXcZOiqrqzUnW0Awes22HdQdvCfiPoWVzuipaVf+Y5HSaT5THToo2xSej90nyKpoTo+2SZOzEiG5vj3gfsCzNPSafT/JufndidFGHdX+rqr4CfKU9STuE5h7eTgIc8Kskh9L0zqgkT6+qT6fpLtrVh8W/TPJHVfXV9mrQDdCc8yTpsifIUTQjIH+5/R1VwE9pemY8u8O6b2Ti9+zLOqz7WZr37RfGZlTVh9orvO/psO5nkmxbVb8YCm8PAC7rsO7YueSz2vfVuTS3MXTp32hulwD4MLAzsLbthXJRV0Wr6j1pbm36K2BfmrwzH/g08A9d1aUZ2Xq4LbfT9JaY1tHie30FTpI2d0l2oDkpWkQzMAD87qTopKoa7o49XXWPBC6uqt87IRg7Ce2o7tuBc6rqC0PzFwLvqY4GmEpyIvD2GviKlXb+A2he5yO7qDtU6wjg72i6ot2v41onDM16XzUDLt2P5nXo7GtXkjyR9U+MrqU5MTq17bLURc0lVbW4i31PUvfhwNtpbhl4Jc1xH03T4+cvqmrab51I8jCaLt/zaT5wemHb9Wsu8Jyq+ufprjlQ+0E0g3qcX5v264pm6muSJqp7WFV1ea/wjB8vzQcQ+1TV97qsuzkc6xZZt6p8+PDhw8cMPIAXWHfLqgvcHfiD2XK81t1yagIvp7kC9GngKmDRwLILt8C6L5tldTf56zybjnVTH69X4CRphgzfU2Nd61rXujNVMzP7dUXW3QLrzqZj3dR1e30PnCRt7pJ8d6JFdPjVI9a1rnX7WXemjpWhrytqu8qemWRPxh8MybrW3Rxrzoq6BjhJ6tZMffWIda1r3X7WnW1fV2TdLbfubDrWTVrXACdJ3fovmiGpLxpekORL1rWuda27GdQEeD6w3iA01QxK8/wkXX7psnW33Lqz6Vg3aV3vgZMkSZKknujyu2IkSZIkSdPIACdJkiRJPWGAkyRJkqSeMMBJkiRJUk8Y4CRpM5TkQ0n+oZ1+XJLLpnHfn0tydDt9TJKvTuO+/zTJOdO1vynUfWySy5P8IsnTx1l+VZKDR9xXJXnABrZjytvO4Gv229ckyd8l+fdR1t2AOtP6/pWk2c6vEZCkzVxVfQV44GTrJXkj8ICq+rNJ9nfYdLQryTzgh8Bd26GSqaqPAh+djv1P0YnAe6vq3TNQe6PM4Gs22Ia3TNe+khQwv6pWtfse6f0rSRqNV+AkaZZIY0v9vb8nsHKmGyFJUte21D/kktQrSQ5IcmGS/01yOnC3gWVPTLJ64Plrkqxp170syZOTLAT+Djiq7Ub4nXbdLyV5c5KvAbcAe7fz/nz98nlvkp8n+X6SJw8sWK/rXJI3JvnP9un/tP/e1NZ8zHCXzCR/mGR5u+/lSf5wYNmXkrwpydfaYzknyc538hr9RZJVSW5IsjTJLu38K4C9gc+27dhmktf6wCTfSHJTkh+3x7710GpPTXJlkuuSvGMw+CZ5YZJLk9yY5Owke05Q56lJLmmPbU2SV0+w3vBrVkle3HYJvSnJyUkyzna7JLk1yY4D8w5o23zXJPsk+WKS69t5H01y7wnaMPhzJcnzklzdbvu6UV+/JGPvie+0P4ujxnn/Prj92d+UZGWSIwaWfag93rPa1+2CJPuM12ZJmq0McJI0w9qT308DHwF2BD4OPHOCdR8IHAc8qqruBTwFuKqqPg+8BTi9qratqocPbPY84FjgXsDV4+z2IOAKYGfgBOCTg6HgTjy+/ffebc1vDLV1R+As4J+BnYB3Amcl2WlgtecCLwDuA2wNTBRyngS8FXg2cP/2OJYAVNU+wDXAH7ftuG2Sdt8OvLI93scATwZeMrTOnwALgEcAi4AXtu1YRBOUnwHMBb4CfGyCOh8A/rL9Of0B8MVJ2jXoacCjgIfRHPNThleoqh8B32D998pzgTOr6jdAaF6zXYAHA7sDb5yscJL9gH+hed/sQvOz221glQlfv6oae088vP1ZnD6077sCnwXOofmZvwz4aPu+HrMY+HtgB2AV8ObJ2ixJs4kBTpJm3qOBuwL/VFW/qaozgeUTrHs7sA2wX5K7VtVVVXXFJPv/UFWtrKp17Yn9sJ8N1D4duAw4fAOPZdDhwOVV9ZG29seA7wN/PLDOB6vqB1V1K3AGsP8E+/pT4NSqurANaK8FHpPmPrwpqapvVdX5bZuuAt4PPGFotbdV1Q1VdQ3wT8Bz2vkvBt5aVZe29/29Bdh/gqtwv6H5OW1XVTdW1YVTaOZJVXVTW/88Jn5dThtrW3uVbnE7j6paVVXnVtVtVbWWJkAPH+d4jgT+q6r+p32t/y9wx9jCEV+/iTwa2LY9vl9X1ReB/+J3ry/Ap6rqm+3r+9E7OXZJmpUMcJI083YB1lRVDcwb70oZ7cAQf01zJeVnSZaMdSW8E9dOsny82pPtcxS78PvHcTWw68DznwxM30Jzcj/pvqrqF8D1Q/saSZJ9k/xXkp8kuZkmhA133Rx8zQZfjz2Bd7fd/24CbqC50jVeO54JPBW4OsmXkzxmCs0c9XX5BE2QvT/NFdE7aK4KkuS+7ftjTXuc/8nvH+d4dmHg+KvqlzSvNe1+R3n97nTfVXXHwLwNfU9I0qxkgJOkmfdjYNeh+5z2mGjlqjqtqv6IJkwU8LaxRRNtMkn98Wr/qJ3+JXCPgWX3m8J+f9S2cdAewJpJtpt0X0nuSdO1b0P29S80VwLnV9V2NF0ih+8x231gevD1uJamW+S9Bx53r6qvDxepquVVtYimq+Cnaa4wTququpGmO+JRNN0nlwyE8bfQ/Iwe2h7nn/H7xzmeHzNw/EnuQfNajxnl9ZvIj4Dds/5gOhv6npCkWckAJ0kz7xvAOuDl7eATzwAOHG/FJA9M8qR2oI5fAbfyu+5tPwXmZeojTd5noPazaO6XWtYuuwhY3C5bQNO9bszatvbeE+x3GbBvkucm2SrJUcB+NF3mpupjwAuS7N8e+1uAC9oufFN1L+Bm4BdJHgT81Tjr/E2SHZLsDrwCGLuX61+B1yZ5CECS7dvXbD1Jtk7z/W7bt91Wb2agG+I0Ow14Ps3P5rSB+fcCfgH8PMmuwN+MuL8zgacl+aP2/swTWf98YbLX76dM/J64gOaq2t+276kn0nSpXTJi2yRp1jPASdIMq6pf0wyKcQxNl7yjgE9OsPo2wEnAdTRdze5Dcz8YNIOfAFyfZCr3W10AzG/3+WbgyKoa6zL3f4F9gBtpBpb4bUCoqlva9b/Wdil89NBxXU8zGMf/oemC97fA06rquim0bWxfX2jb8gmaK0T70NzvtSFeTXO16n+Bf+N34WzQZ4Bv0QTYs2gGJKGqPkVzxXNJ233we8BE36v3POCqdr0X09zH14WlND+/n1TVdwbm/z3NICw/pzmGid5T66mqlcBLaX7WP6b52a8eWGWy1++NwIfb98Szh/b9a5rAdhjN++19wPOr6vujtE2SBFn/tgdJkiRJ0ubKK3CSJEmS1BMGOEmSJEnqCQOcJEmSJPWEAU6SJEmSemKrmW7AsJ133rnmzZs3082QJEmSpBnxrW9967qqmjvess0uwM2bN48VK1bMdDMkSZIkaUYkuXqiZXahlCRJkqSeMMBJkiRJUk8Y4CRJkiSpJwxwkiRJktQTBjhJkiRJ6gkDnCRJkiT1hAFOkiRJknrCACdJkiRJPbHZfZG3JEmSJPXBvOPP2uBtrzrp8A3azitwkiRJktQTBjhJkiRJ6gkDnCRJkiT1hAFOkiRJknrCACdJkiRJPWGAkyRJkqSeMMBJkiRJUk8Y4CRJkiSpJwxwkiRJktQTBjhJkiRJ6gkDnCRJkiT1xEgBLsnCJJclWZXk+HGWvzjJxUkuSvLVJPsNLHttu91lSZ4ynY2XJEmSpNlk0gCXZA5wMnAYsB/wnMGA1jqtqh5aVfsDbwfe2W67H7AYeAiwEHhfuz9JkiRJ0hSNcgXuQGBVVV1ZVb8GlgCLBleoqpsHnt4TqHZ6EbCkqm6rqh8Cq9r9SZIkSZKmaKsR1tkVuHbg+WrgoOGVkrwUeBWwNfCkgW3PH9p213G2PRY4FmCPPfYYpd2SJEmSNOtM2yAmVXVyVe0DvAZ4/RS3PaWqFlTVgrlz505XkyRJkiRpizJKgFsD7D7wfLd23kSWAE/fwG0lSZIkSRMYJcAtB+Yn2SvJ1jSDkiwdXCHJ/IGnhwOXt9NLgcVJtkmyFzAf+ObGN1uSJEmSZp9J74GrqnVJjgPOBuYAp1bVyiQnAiuqailwXJKDgd8ANwJHt9uuTHIGcAmwDnhpVd3e0bFIkiRJ0hZtlEFMqKplwLKheW8YmH7FnWz7ZuDNG9pASZIkSVJj2gYxkSRJkiR1ywAnSZIkST1hgJMkSZKknjDASZIkSVJPGOAkSZIkqScMcJIkSZLUEwY4SZIkSeoJA5wkSZIk9YQBTpIkSZJ6wgAnSZIkST1hgJMkSZKknjDASZIkSVJPGOAkSZIkqScMcJIkSZLUEwY4SZIkSeoJA5wkSZIk9YQBTpIkSZJ6wgAnSZIkST1hgJMkSZKknjDASZIkSVJPGOAkSZIkqSdGCnBJFia5LMmqJMePs/xVSS5J8t0k/51kz4Fltye5qH0snc7GS5IkSdJsstVkKySZA5wMHAKsBpYnWVpVlwys9m1gQVXdkuSvgLcDR7XLbq2q/ae32ZIkSZI0+4xyBe5AYFVVXVlVvwaWAIsGV6iq86rqlvbp+cBu09tMSZIkSdIoAW5X4NqB56vbeRN5EfC5ged3S7IiyflJnj71JkqSJEmSYIQulFOR5M+ABcATBmbvWVVrkuwNfDHJxVV1xdB2xwLHAuyxxx7T2SRJkiRJ2mKMcgVuDbD7wPPd2nnrSXIw8DrgiKq6bWx+Va1p/70S+BJwwPC2VXVKVS2oqgVz586d0gFIkiRJ0mwxSoBbDsxPsleSrYHFwHqjSSY5AHg/TXj72cD8HZJs007vDDwWGBz8RJIkSZI0okm7UFbVuiTHAWcDc4BTq2plkhOBFVW1FHgHsC3w8SQA11TVEcCDgfcnuYMmLJ40NHqlJEmSJGlEI90DV1XLgGVD894wMH3wBNt9HXjoxjRQkiRJktQY6Yu8JUmSJEkzzwAnSZIkST1hgJMkSZKknjDASZIkSVJPGOAkSZIkqScMcJIkSZLUEwY4SZIkSeoJA5wkSZIk9YQBTpIkSZJ6wgAnSZIkST1hgJMkSZKknjDASZIkSVJPGOAkSZIkqScMcJIkSZLUEwY4SZIkSeoJA5wkSZIk9YQBTpIkSZJ6wgAnSZIkST1hgJMkSZKknjDASZIkSVJPGOAkSZIkqScMcJIkSZLUEyMFuCQLk1yWZFWS48dZ/qoklyT5bpL/TrLnwLKjk1zePo6ezsZLkiRJ0mwyaYBLMgc4GTgM2A94TpL9hlb7NrCgqh4GnAm8vd12R+AE4CDgQOCEJDtMX/MlSZIkafYY5QrcgcCqqrqyqn4NLAEWDa5QVedV1S3t0/OB3drppwDnVtUNVXUjcC6wcHqaLkmSJEmzyygBblfg2oHnq9t5E3kR8LmpbJvk2CQrkqxYu3btCE2SJEmSpNlnWgcxSfJnwALgHVPZrqpOqaoFVbVg7ty509kkSZIkSdpijBLg1gC7DzzfrZ23niQHA68Djqiq26ayrSRJkiRpcqMEuOXA/CR7JdkaWAwsHVwhyQHA+2nC288GFp0NHJpkh3bwkkPbeZIkSZKkKdpqshWqal2S42iC1xzg1KpameREYEVVLaXpMrkt8PEkANdU1RFVdUOSN9GEQIATq+qGTo5EkiRJkrZwkwY4gKpaBiwbmveGgemD72TbU4FTN7SBkiRJkqTGtA5iIkmSJEnqjgFOkiRJknrCACdJkiRJPWGAkyRJkqSeMMBJkiRJUk8Y4CRJkiSpJwxwkiRJktQTBjhJkiRJ6gkDnCRJkiT1hAFOkiRJknrCACdJkiRJPWGAkyRJkqSeMMBJkiRJUk8Y4CRJkiSpJwxwkiRJktQTBjhJkiRJ6gkDnCRJkiT1hAFOkiRJknrCACdJkiRJPWGAkyRJkqSeMMBJkiRJUk+MFOCSLExyWZJVSY4fZ/njk1yYZF2SI4eW3Z7kovaxdLoaLkmSJEmzzVaTrZBkDnAycAiwGlieZGlVXTKw2jXAMcCrx9nFrVW1/8Y3VZIkSZJmt0kDHHAgsKqqrgRIsgRYBPw2wFXVVe2yOzpooyRJkiSJ0bpQ7gpcO/B8dTtvVHdLsiLJ+UmePt4KSY5t11mxdu3aKexakiRJkmaPTTGIyZ5VtQB4LvBPSfYZXqGqTqmqBVW1YO7cuZugSZIkSZLUP6MEuDXA7gPPd2vnjaSq1rT/Xgl8CThgCu2TJEmSJLVGCXDLgflJ9kqyNbAYGGk0ySQ7JNmmnd4ZeCwD985JkiRJkkY3aYCrqnXAccDZwKXAGVW1MsmJSY4ASPKoJKuBZwHvT7Ky3fzBwIok3wHOA04aGr1SkiRJkjSiUUahpKqWAcuG5r1hYHo5TdfK4e2+Djx0I9soSZIkSWLTDGIiSZIkSZoGBjhJkiRJ6gkDnCRJkiT1hAFOkiRJknrCACdJkiRJPWGAkyRJkqSeMMBJkiRJUk8Y4CRJkiSpJwxwkiRJktQTBjhJkiRJ6gkDnCRJkiT1hAFOkiRJknrCACdJkiRJPWGAkyRJkqSeMMBJkiRJUk8Y4CRJkiSpJwxwkiRJktQTBjhJkiRJ6gkDnCRJkiT1hAFOkiRJknrCACdJkiRJPWGAkyRJkqSeGCnAJVmY5LIkq5IcP87yxye5MMm6JEcOLTs6yeXt4+jpargkSZIkzTaTBrgkc4CTgcOA/YDnJNlvaLVrgGOA04a23RE4ATgIOBA4IckOG99sSZIkSZp9RrkCdyCwqqqurKpfA0uARYMrVNVVVfVd4I6hbZ8CnFtVN1TVjcC5wMJpaLckSZIkzTqjBLhdgWsHnq9u541ipG2THJtkRZIVa9euHXHXkiRJkjS7bBaDmFTVKVW1oKoWzJ07d6abI0mSJEmbpVEC3Bpg94Hnu7XzRrEx20qSJEmSBowS4JYD85PslWRrYDGwdMT9nw0cmmSHdvCSQ9t5kiRJkqQpmjTAVdU64Dia4HUpcEZVrUxyYpIjAJI8Kslq4FnA+5OsbLe9AXgTTQhcDpzYzpMkSZIkTdFWo6xUVcuAZUPz3jAwvZyme+R4254KnLoRbZQkSZIkMWKA25zMO/6sDd72qpMOn8aWSJIkSdKmtVmMQilJkiRJmpwBTpIkSZJ6wgAnSZIkST1hgJMkSZKknjDASZIkSVJPGOAkSZIkqScMcJIkSZLUEwY4SZIkSeoJA5wkSZIk9cRWM90Aacy848/a4G2vOunwaWyJJEmStHnyCpwkSZIk9YQBTpIkSZJ6wgAnSZIkST1hgJMkSZKknnAQE0mSesCBniRJ4BU4SZIkSeoNA5wkSZIk9YQBTpIkSZJ6wgAnSZIkST1hgJMkSZKknhhpFMokC4F3A3OAf6+qk4aWbwP8B/BI4HrgqKq6Ksk84FLgsnbV86vqxdPU9k1qto3+NduOV5IkSeqDSQNckjnAycAhwGpgeZKlVXXJwGovAm6sqgckWQy8DTiqXXZFVe0/vc2WpM2bH4JIkqQujNKF8kBgVVVdWVW/BpYAi4bWWQR8uJ0+E3hykkxfMyVJkiRJowS4XYFrB56vbueNu05VrQN+DuzULtsrybeTfDnJ48YrkOTYJCuSrFi7du2UDkCSJEmSZouuBzH5MbBHVR0AvAo4Lcl2wytV1SlVtaCqFsydO7fjJkmSJElSP40yiMkaYPeB57u188ZbZ3WSrYDtgeurqoDbAKrqW0muAPYFVmxswyVJkqab969K2tyNEuCWA/OT7EUT1BYDzx1aZylwNPAN4Ejgi1VVSeYCN1TV7Un2BuYDV05b6yVNmScnkiRJ/TVpgKuqdUmOA86m+RqBU6tqZZITgRVVtRT4APCRJKuAG2hCHsDjgROT/Aa4A3hxVd3QxYFIkiRJ0pZupO+Bq6plwLKheW8YmP4V8KxxtvsE8ImNbKMkSZIkie4HMZEkSZIkTZORrsBJkiRpy+N90VL/eAVOkiRJknrCK3CbOT8ZkyRJkjTGK3CSJEmS1BNegZMkSZsde6BI0vgMcJK2aJ4ESpKkLYkBTrOeJ/iSJEnqCwOcNEMMjpIkSZoqA5wkSZKkaeEH1N0zwEmSJGlWMFxsuWbTz9avEZAkSZKknjDASZIkSVJPGOAkSZIkqSe8B06StiCz6R4A2PDj7eOxSpIEBjhJkqZktoVkSdLmxQAnaZPwpFeSJGnjGeAkSZKkLYwfnG65HMREkiRJknrCACdJkiRJPWGAkyRJkqSeGOkeuCQLgXcDc4B/r6qThpZvA/wH8EjgeuCoqrqqXfZa4EXA7cDLq+rsaWu9JGmz4L0WkiRtGpMGuCRzgJOBQ4DVwPIkS6vqkoHVXgTcWFUPSLIYeBtwVJL9gMXAQ4BdgC8k2beqbp/uA5EkSeorPwSRNKpRrsAdCKyqqisBkiwBFgGDAW4R8MZ2+kzgvUnSzl9SVbcBP0yyqt3fN6an+ZIkqUsGC3VhNr2vZtOxatMY5R64XYFrB56vbueNu05VrQN+Duw04raSJEmSpBGkqu58heRIYGFV/Xn7/HnAQVV13MA632vXWd0+vwI4iOaq3PlV9Z/t/A8An6uqM4dqHAsc2z59IHDZBh7PzsB1G7jtxrCuda3bz7qz6Vita13r9rOmda1r3dlZd8+qmjveglG6UK4Bdh94vls7b7x1VifZCtieZjCTUbalqk4BThmhLXcqyYqqWrCx+7Guda07O+rOpmO1rnWt28+a1rWuda07bJQulMuB+Un2SrI1zaAkS4fWWQoc3U4fCXyxmkt7S4HFSbZJshcwH/jm9DRdkiRJkmaXSa/AVdW6JMcBZ9N8jcCpVbUyyYnAiqpaCnwA+Eg7SMkNNCGPdr0zaAY8WQe81BEoJUmSJGnDjPQ9cFW1DFg2NO8NA9O/Ap41wbZvBt68EW2cio3uhmld61p3VtWdTcdqXetat581rWtd61p3PZMOYiJJkiRJ2jyMcg+cJEmSJGkzYICTJEmSpJ4wwEmSJElST4w0iMnmKsmDgEXAru2sNcDSqrp05lrVnfZ4dwUuqKpfDMxfWFWf77DugUBV1fIk+wELge+3g9tsEkn+o6qev6nqDdT9I+BA4HtVdU6HdQ4CLq2qm5PcHTgeeATNCK5vqaqfd1Dz5cCnqura6d73JHXHvo7kR1X1hSTPBf4QuBQ4pap+02HtvYFn0Hw/5e3AD4DTqurmrmpKkrQ5SnKfqvrZTLdjU0iyU1VdP9PtmC69vQKX5DXAEiA03y33zXb6Y0mOn6E2vaDDfb8c+AzwMuB7SRYNLH5Lh3VPAP4Z+JckbwXeC9wTOD7J6zqquXTo8VngGWPPu6g5UPubA9N/QXO89wJO6Ph9dSpwSzv9bmB74G3tvA92VPNNwAVJvpLkJUnmdlRn2AeBw4FXJPkIzQi2FwCPAv69q6Lt/6F/Be7W1tqGJsidn+SJXdXV7JPkPjPdhk0pyU4z3QZtvCTbJzkpyfeT3JDk+iSXtvPuPUNt+lyH+94uyVuTfKT9IHFw2fs6rHu/JP+S5OQkOyV5Y5KLk5yR5P4d1t1x6LET8M0kOyTZsaOaCwemt0/ygSTfTXJakvt2UbOtdVKSndvpBUmupDnfuTrJEzqse2GS1yfZp6sav1VVvXzQfHJ+13Hmbw1cPkNtuqbDfV8MbNtOzwNWAK9on3+747pzgHsANwPbtfPvDny3o5oXAv8JPBF4Qvvvj9vpJ3T8M/z2wPRyYG47fU/g4g7rXjp4/EPLLurqWGk+xDmU5rsc1wKfB44G7tXhsX63/Xcr4KfAnPZ5unpPtfu/eKDWPYAvtdN7dPx/aHvgJOD7NN+TeT3N1caTgHt3VXeSNn2uw31vB7wV+Ajw3KFl7+uw7v2AfwFOBnYC3tj+zM8A7t9h3R2HHjsBVwE7ADt2WHfh0HvsA8B3gdOA+3ZY9yRg53Z6AXAlsAq4usvfz+3fhdcD+3RVY4K6C4Dz2r9JuwPnAj9v/z4c0FHNbYETgZVtrbXA+cAxHR/r2cBrgPsNzLtfO++cDus+YoLHI4Efd1j3E+37+enA0vb5NmPvtw7rfp7mw/jj2/+zr2nfWy8DPtNh3TuAHw49ftP+e2VHNS8cmP534B+APYFXAp/u8FgvHpg+D3hUO70vzXdYd1X3h8D/A66hubD0SmCXLmr1uQvlHcAuNH80Bt2/XdaJJN+daBHQ2acJwF2q7TZZVVe1VwzOTLJnW7sr66r58vVbklxRbVezqro1SVev8wLgFcDrgL+pqouS3FpVX+6o3qC7JNmBJtikqtYCVNUvk6zrsO73krygqj4IfCfJgqpakWRfml+wXaiqugM4BzgnyV2Bw4Dn0PwC6uqK3F3abpT3pAlS29MEm22Au3ZUc8xWNF0nt6E5SaKqrmmPvStnAF8EnlhVP4HmE1iaoHwGTYCedkkeMdEiYP8uarY+CFxOczL0wiTPpAlytwGP7rDuh4CzaN5X5wEfBZ5Kc3L2rzTd7btwHb//d2hXmsBRwN4d1X0LzYkgwD/SfMj1xzRdhN9Pc9xdOLyqxnojvAM4qpru9fvShMcFHdXdAbg3cF6SnwAfA06vqh91VG/M+4AT2tpfB15ZVYckeXK77DEd1Pwo8CngKcCzad7TS4DXJ9m3qv6ug5oA86rqbYMz2t9Zb0vywo5qQhOGv8z45zL37rDuPlX1zHb6022voi8mOaLDmtB8wPIegCQvGXjN35PkRR3W/RvgEJrzqovb+j+sqr06rDloQVXt306/K8nRHdbaKslWVbUOuHtVLQeoqh8k2abDujdW1auBVyd5HM351IVJLgU+VlXT931wXaXQrh8092GtAj5H8wV5p9D8MVvFwCeTHdT9Kc3Jz55Dj3k09/R0VfeLwP5D87YC/gO4vcO6FwD3aKfvMjB/ezr8hKqtsRvwcZpujJ1d3RyqeRXNJ8o/bP+9fzt/Wzq6Ejbwen4IuKJ9zX/T1v8y8PCOan77Tpbdo8NjfWV7bFcDLwf+G/g3mqslJ3RY9xU0n3b+G83VsBe08+cC/9Nh3cs2ZNk01L29/b1x3jiPWzuse9HQ89cBX6O5MtXlp9rfHpi+5s7aNM11/0/7t+ehA/N+2FW9gRqDn2wPv+ZdHu+lwFbt9PlDy7rspTB4vI+jCU8/ad/Px3ZY99sD08Pvq293VPM7Q8+Xt//eheb+866O9Rzgbxm4gkvzwfRrgC90WPd7wPwJll3bYd1LGTivaecdQ3Pl8+oO635nYPofhpZ19n+o3f/YedU7aW4P6eTK20C91cCr2t+TV9J+/3S7rMseNy9r389PoumN8W6aXlx/D3ykw7q/9zeOphfbQuCD01mrt1fgqurz7Sd+B7L+ICbLq7li1JX/ounKeNHwgiRf6rDu84H1rgBV88nC85O8v8O6j6/mk3OquVoz5q40VxA6U1WrgWclOZym+2bnqmreBIvuAP6kw7o/B45Jsh2wF004X11VP+2qJnDUnbTnlomWbayqeleS09vpHyX5D+Bg4N+q6pt3vvVG1X13ki8ADwb+saq+385fCzy+q7rA1Un+Fvjw2M+z7ft/DNDlADKXAn9ZVZcPL0jSZd1tktxl7PdFVb05yRrgf2ivenZk8J7u/xhaNqerolX1j+37+V3t63oCzZW3rt0nyatorlpslyTVni3Q7f3t7wOWJTkJ+HySdwOfpDlRuqjDur9VVV8BvpLkZTRXFI6i+RC3C79KcijNh2yV5OlV9en2PpquzjV+meSPquqr7dWgG6D5G5ykyx43R9F06/ty+zuqaD60XkpzJbArb2Ti9+zLOqz7WZr37RfGZlTVh9orvO/psO5nkmxbVb+oqtePzUzyAOCyDusOnlcdQdMd+B5d1qP5wPRe7fSHgZ2BtW0vlIu6KlpV70lyMfBXNN0mtwLmA5+m6cbZlR+M05bbaT7km9bBBvO73/eSpOnWdsk9nqYL39jgFmMnRSdV1Y0d1T2S5tPc3zshGDsJ7aju22nul/nC0PyFwHuqan5HdU8E3l4DI/S28x9A8zof2UXdoVpHAH9H0xXtfh3XOmFo1vuqauzE6O3V4ai9bRf+wROja2lOjE5tP1jsouaSqlrcxb4nqftw4O00H+K9kua4j6b5wPgvqurrHdR8GM39QvNprga9sJquX3OB51TVP093zYHaD6K5SnN+bdrRrmdqlO2J6h5WVV0OoDLjx0vzAcQ+VfW9LutuDse6Rdbt6jKiDx8+fPi48wdtN07rbjl1aQZ4+oPZcrzW3XJq0nRpv4wmjF8FLBpY1mX355mq+7JZVneTv86z6Vg39fF6BU6SZkiSa6pqD+ta17rWnemabZezx1TVL5LMA86kuV/o3Um+XVUHWNe6m3vN2VK3t/fASVIfZIZGrrWuda3bz7ozdazM3GjX1t1y686mY92kdQ1wktSt+9IMBz58r1tohiW3rnWta92Zrgnw0yT7VztIW3sV4WnAqcBDrWvdntScFXUNcJLUrZkauda61rVuP+vOttGurbvl1p1Nx7pJ63oPnCRJkiT1RJffFSNJkiRJmkYGOEmSJEnqCQOcJEmSJPWEAU6SJEmSesIAJ0lbsCQfSvIP7fTjklw2jfv+XJKj2+ljknx1Gvf9p0nOma79TaHuY5NcnuQXSZ4+zvKrkhw84r4qyQM2sB1T3namXjNJ0qZlgJOkWaKqvlJVD5xsvSRvTPKfI+zvsKr68Ma2K8m8NrD89qttquqjVXXoxu57A5wIvLeqtq2qT89A/Q22Ma/ZqD/zEfe1wcFVkjQ5A5wkaUrS2FL/fuwJrJzpRkiSNJEt9Q+wJM1KSQ5IcmGS/01yOnC3gWVPTLJ64Plrkqxp170syZOTLAT+Djiq7Ub4nXbdLyV5c5KvAbcAe7fz/nz98nlvkp8n+X6SJw8sWK/r4dAVn/9p/72prfmY4S6ZSf4wyfJ238uT/OHAsi8leVOSr7XHck6Sne/kNfqLJKuS3JBkaZJd2vlXAHsDn23bsc0kr/WBSb6R5KYkP26Pfeuh1Z6a5Mok1yV5x2DwTfLCJJcmuTHJ2Un2nKDOU5Nc0h7bmiSvnmC94deskry47RJ6U5KTk2Sc7Sb6mW+f5APtsa1J8g9J5rTLHpDky+3P47r2vUaSsZ/ld9p9HXVnr6EkaeoMcJK0hWjDw6eBjwA7Ah8HnjnBug8EjgMeVVX3Ap4CXFVVnwfeApzediN8+MBmzwOOBe4FXD3Obg8CrgB2Bk4APplkxxGa/vj233u3Nb8x1NYdgbOAfwZ2At4JnJVkp4HVngu8ALgPsDUwUch5EvBW4NnA/dvjWAJQVfsA1wB/3LbjtknafTvwyvZ4HwM8GXjJ0Dp/AiwAHgEsAl7YtmMRTWh6BjAX+ArwsQnqfAD4y/bn9AfAFydp16CnAY8CHkZzzE8ZXuFOfuYfAtYBDwAOAA4FxgL7m4BzgB2A3YD3tPsa+1k+vN3X6VNoqyRpBAY4SdpyPBq4K/BPVfWbqjoTWD7BurcD2wD7JblrVV1VVVdMsv8PVdXKqlpXVb8ZZ/nPBmqfDlwGHL6BxzLocODyqvpIW/tjwPeBPx5Y54NV9YOquhU4A9h/gn39KXBqVV3YBrTXAo9JMm+qjaqqb1XV+W2brgLeDzxhaLW3VdUNVXUN8E/Ac9r5LwbeWlWXVtU6mgC1/wRX4X5D83ParqpurKoLp9DMk6rqprb+eUz8uqwnyX2BpwJ/XVW/rKqfAe8CFg+0aU9gl6r6VVVN2wA2kqQ7Z4CTpC3HLsCaqqqBeeNdKaOqVgF/DbwR+FmSJWNdCe/EtZMsH6/2ZPscxS78/nFcDew68PwnA9O3ANuOsq+q+gVw/dC+RpJk3yT/leQnSW6mCWHDXTcHX7PB12NP4N1t18abgBuATNCOZ9KEqavbbouPmUIzR31dhu1J82HAjwfa+H6aK5wAf9u295tJViZ54RTaJEnaCAY4Sdpy/BjYdeg+pz0mWrmqTquqP6I5WS/gbWOLJtpkkvrj1f5RO/1L4B4Dy+43hf3+qG3joD2ANZNsN+m+ktyTplvmhuzrX2iuBM6vqu1oukQO32O2+8D04OtxLU23yHsPPO5eVV8fLlJVy6tqEU14+jTNFcbpNvwzuBa4Ddh5oH3bVdVD2jb9pKr+oqp2Af4SeF8ceVKSNgkDnCRtOb5Bc8/Sy5PcNckzgAPHWzHJA5M8qR2o41fArcAd7eKfAvMy9ZEm7zNQ+1nAg4Fl7bKLgMXtsgXAkQPbrW1r7z3BfpcB+yZ5bpKt2oEx9gP+a4rtg+Y+sxck2b899rcAF7RdIKfqXsDNwC+SPAj4q3HW+ZskOyTZHXgFMHZP2L8Cr03yEPjtgCHPGt44ydZpvt9t+7bb6s387uc0ndb7mVfVj2nucfvHJNsluUuSfZI8oW3Xs5Ls1m57I00AHHz/TPSzlCRtJAOcJG0hqurXNINiHEPTJe8o4JMTrL4NcBJwHU03u/vQ3A8GzeAnANcnmcr9VhcA89t9vhk4sqqub5f9X2AfmpP9vwdOG2j3Le36X2u76z166LiupxmM4//QdHf8W+BpVXXdFNo2tq8vtG35BM0Vy3343X1dU/VqmsFT/hf4N34XzgZ9BvgWTYA9i2ZAEqrqUzRXPJe03S+/Bxw2QZ3nAVe1672Y5j6+6Tbez/z5NAPCXELzczuTZuAXaAZGuSDJL4ClwCuq6sp22RuBD7c/y2d30FZJmtWy/u0KkiRJkqTNlVfgJEmSJKknDHCSJEmS1BMGOEmSJEnqCQOcJEmSJPXEVjPdgGE777xzzZs3b6abIUmSJEkz4lvf+tZ1VTV3vGWbXYCbN28eK1asmOlmSJIkSdKMSHL1RMvsQilJkiRJPWGAkyRJkqSeMMBJkiRJUk8Y4CRJkiSpJwxwkiRJktQTBjhJkiRJ6gkDnCRJkiT1hAFOkiRJknpis/sib0mSJEnqg3nHn7XB21510uEbtJ1X4CRJkiSpJwxwkiRJktQTBjhJkiRJ6gkDnCRJkiT1hAFOkiRJknrCACdJkiRJPWGAkyRJkqSeMMBJkiRJUk8Y4CRJkiSpJwxwkiRJktQTBjhJkiRJ6gkDnCRJkiT1hAFOkiRJknrCACdJkiRJPTFSgEuyMMllSVYlOX6c5S9OcnGSi5J8Ncl+A8te2253WZKnTGfjJUmSJGk2mTTAJZkDnAwcBuwHPGcwoLVOq6qHVtX+wNuBd7bb7gcsBh4CLATe1+5PkiRJkjRFo1yBOxBYVVVXVtWvgSXAosEVqurmgaf3BKqdXgQsqarbquqHwKp2f5IkSZKkKdpqhHV2Ba4deL4aOGh4pSQvBV4FbA08aWDb84e23XWcbY8FjgXYY489Rmm3JEmSJM060zaISVWdXFX7AK8BXj/FbU+pqgVVtWDu3LnT1SRJkiRJ2qKMEuDWALsPPN+tnTeRJcDTN3BbSZIkSdIERglwy4H5SfZKsjXNoCRLB1dIMn/g6eHA5e30UmBxkm2S7AXMB7658c2WJEmSpNln0nvgqmpdkuOAs4E5wKlVtTLJicCKqloKHJfkYOA3wI3A0e22K5OcAVwCrANeWlW3d3QskiRJkrRFG2UQE6pqGbBsaN4bBqZfcSfbvhl484Y2UJIkSZLUmLZBTCRJkiRJ3TLASZIkSVJPGOAkSZIkqScMcJIkSZLUEwY4SZIkSeoJA5wkSZIk9YQBTpIkSZJ6wgAnSZIkST1hgJMkSZKknjDASZIkSVJPGOAkSZIkqScMcJIkSZLUEwY4SZIkSeoJA5wkSZIk9YQBTpIkSZJ6wgAnSZIkST1hgJMkSZKknjDASZIkSVJPGOAkSZIkqScMcJIkSZLUEwY4SZIkSeqJkQJckoVJLkuyKsnx4yx/VZJLknw3yX8n2XNg2e1JLmofS6ez8ZIkSZI0m2w12QpJ5gAnA4cAq4HlSZZW1SUDq30bWFBVtyT5K+DtwFHtslurav/pbbYkSZIkzT6jXIE7EFhVVVdW1a+BJcCiwRWq6ryquqV9ej6w2/Q2U5IkSZI0SoDbFbh24Pnqdt5EXgR8buD53ZKsSHJ+kqePt0GSY9t1Vqxdu3aEJkmSJEnS7DNpF8qpSPJnwALgCQOz96yqNUn2Br6Y5OKqumJwu6o6BTgFYMGCBTWdbZIkSZKkLcUoV+DWALsPPN+tnbeeJAcDrwOOqKrbxuZX1Zr23yuBLwEHbER7JUmSJGnWGiXALQfmJ9krydbAYmC90SSTHAC8nya8/Wxg/g5JtmmndwYeCwwOfiJJkiRJGtGkXSiral2S44CzgTnAqVW1MsmJwIqqWgq8A9gW+HgSgGuq6gjgwcD7k9xBExZPGhq9UpIkSZI0opHugauqZcCyoXlvGJg+eILtvg48dGMaKEmSJElqjPRF3pIkSZKkmWeAkyRJkqSeMMBJkiRJUk8Y4CRJkiSpJwxwkiRJktQTBjhJkiRJ6gkDnCRJkiT1hAFOkiRJknrCACdJkiRJPWGAkyRJkqSeMMBJkiRJUk8Y4CRJkiSpJwxwkiRJktQTBjhJkiRJ6gkDnCRJkiT1hAFOkiRJknrCACdJkiRJPWGAkyRJkqSeMMBJkiRJUk8Y4CRJkiSpJwxwkiRJktQTIwW4JAuTXJZkVZLjx1n+qiSXJPlukv9OsufAsqOTXN4+jp7OxkuSJEnSbDJpgEsyBzgZOAzYD3hOkv2GVvs2sKCqHgacCby93XZH4ATgIOBA4IQkO0xf8yVJkiRp9hjlCtyBwKqqurKqfg0sARYNrlBV51XVLe3T84Hd2umnAOdW1Q1VdSNwLrBwepouSZIkSbPLKAFuV+Dageer23kTeRHwualsm+TYJCuSrFi7du0ITZIkSZKk2WdaBzFJ8mfAAuAdU9muqk6pqgVVtWDu3LnT2SRJkiRJ2mKMEuDWALsPPN+tnbeeJAcDrwOOqKrbprKtJEmSJGlyowS45cD8JHsl2RpYDCwdXCHJAcD7acLbzwYWnQ0cmmSHdvCSQ9t5kiRJkqQp2mqyFapqXZLjaILXHODUqlqZ5ERgRVUtpekyuS3w8SQA11TVEVV1Q5I30YRAgBOr6oZOjkSSJEmStnCTBjiAqloGLBua94aB6YPvZNtTgVM3tIGSJEmSpMa0DmIiSZIkSeqOAU6SJEmSesIAJ0mSJEk9YYCTJEmSpJ4wwEmSJElSTxjgJEmSJKknDHCSJEmS1BMGOEmSJEnqCQOcJEmSJPWEAU6SJEmSesIAJ0mSJEk9YYCTJEmSpJ4wwEmSJElSTxjgJEmSJKknDHCSJEmS1BMGOEmSJEnqCQOcJEmSJPWEAU6SJEmSesIAJ0mSJEk9YYCTJEmSpJ4wwEmSJElST4wU4JIsTHJZklVJjh9n+eOTXJhkXZIjh5bdnuSi9rF0uhouSZIkSbPNVpOtkGQOcDJwCLAaWJ5kaVVdMrDaNcAxwKvH2cWtVbX/xjdVkiRJkma3SQMccCCwqqquBEiyBFgE/DbAVdVV7bI7OmijJEmSJInRulDuClw78Hx1O29Ud0uyIsn5SZ4+3gpJjm3XWbF27dop7FqSJEmSZo9NMYjJnlW1AHgu8E9J9hleoapOqaoFVbVg7ty5m6BJkiRJktQ/owS4NcDuA893a+eNpKrWtP9eCXwJOGAK7ZMkSZIktUYJcMuB+Un2SrI1sBgYaTTJJDsk2aad3hl4LAP3zkmSJEmSRjdpgKuqdcBxwNnApcAZVbUyyYlJjgBI8qgkq4FnAe9PsrLd/MHAiiTfAc4DThoavVKSJEmSNKJRRqGkqpYBy4bmvWFgejlN18rh7b4OPHQj2yhJkiRJYtMMYiJJkiRJmgYGOEmSJEnqCQOcJEmSJPWEAU6SJEmSesIAJ0mSJEk9YYCTJEmSpJ4wwEmSJElSTxjgJEmSJKknDHCSJEmS1BMGOEmSJEnqCQOcJEmSJPWEAU6SJEmSesIAJ0mSJEk9YYCTJEmSpJ4wwEmSJElSTxjgJEmSJKknDHCSJEmS1BMGOEmSJEnqCQOcJEmSJPWEAU6SJEmSesIAJ0mSJEk9MVKAS7IwyWVJViU5fpzlj09yYZJ1SY4cWnZ0ksvbx9HT1XBJkiRJmm0mDXBJ5gAnA4cB+wHPSbLf0GrXAMcApw1tuyNwAnAQcCBwQpIdNr7ZkiRJkjT7jHIF7kBgVVVdWVW/BpYAiwZXqKqrquq7wB1D2z4FOLeqbqiqG4FzgYXT0G5JkiRJmnVGCXC7AtcOPF/dzhvFSNsmOTbJiiQr1q5dO+KuJUmSJGl22SwGMamqU6pqQVUtmDt37kw3R5IkSZI2S6MEuDXA7gPPd2vnjWJjtpUkSZIkDRglwC0H5ifZK8nWwGJg6Yj7Pxs4NMkO7eAlh7bzJEmSJElTtNVkK1TVuiTH0QSvOcCpVbUyyYnAiqpamuRRwKeAHYA/TvL3VfWQqrohyZtoQiDAiVV1w8Y0eN7xZ23wtleddPjGlJYkSZKkGTVpgAOoqmXAsqF5bxiYXk7TPXK8bU8FTt2INkqSJEmS2EwGMZEkSZIkTc4AJ0mSJEk9YYCTJEmSpJ4wwEmSJElSTxjgJEmSJKknDHCSJEmS1BMGOEmSJEnqCQOcJEmSJPWEAU6SJEmSesIAJ0mSJEk9YYCTJEmSpJ4wwEmSJElSTxjgJEmSJKknDHCSJEmS1BMGOEmSJEnqia1mugHSmHnHn7XB21510uHT2BJJ2vz4O1KSBF6BkyRJkqTeMMBJkiRJUk8Y4CRJkiSpJ7wHTuPyXgtJkiRp8+MVOEmSJEnqiZECXJKFSS5LsirJ8eMs3ybJ6e3yC5LMa+fPS3Jrkovax79Oc/slSZIkadaYtAtlkjnAycAhwGpgeZKlVXXJwGovAm6sqgckWQy8DTiqXXZFVe0/vc2WJEmSpNlnlHvgDgRWVdWVAEmWAIuAwQC3CHhjO30m8N4kmcZ2Spom3t8oSZLUX6MEuF2BaweerwYOmmidqlqX5OfATu2yvZJ8G7gZeH1VfWW4QJJjgWMB9thjjykdgCRtjgzKkiSpC12PQvljYI+quj7JI4FPJ3lIVd08uFJVnQKcArBgwYLquE0bxJMxSZIkSTNtlEFM1gC7DzzfrZ037jpJtgK2B66vqtuq6nqAqvoWcAWw78Y2WpIkSZJmo1EC3HJgfpK9kmwNLAaWDq2zFDi6nT4S+GJVVZK57SAoJNkbmA9cOT1NlyRJkqTZZdIulO09bccBZwNzgFOramWSE4EVVbUU+ADwkSSrgBtoQh7A44ETk/wGuAN4cVXd0MWBSJIkbSxvmZC0uRvpHriqWgYsG5r3hoHpXwHPGme7TwCf2Mg2SpIkSZIY8Yu8JUmSJEkzzwAnSZIkST3R9dcIaCPZF1+SJEnSGAOcJEnSLOUHxVL/2IVSkiRJknrCACdJkiRJPWEXSkmStNmxa58kjc8AJ80QT042DV9nSZI2Hf/uds8AJ0mSpFnBcLHlmk0/WwOcZr3Z9B9ekiRJ/WaAk6QtiB9ISJK0ZXMUSkmSJEnqCQOcJEmSJPWEXSglSb21oV1G7S4qSeorr8BJkiRJUk94BU7SJuHgGtpS+F6WJM0kr8BJkiRJUk8Y4CRJkiSpJ+xCKUmSJG1h7O695fIKnCRJkiT1hFfgJEkbzU96JUnaNEa6ApdkYZLLkqxKcvw4y7dJcnq7/IIk8waWvbadf1mSp0xj2yVJkiRpVpn0ClySOcDJwCHAamB5kqVVdcnAai8CbqyqByRZDLwNOCrJfsBi4CHALsAXkuxbVbdP94FIkiT11Wy7ij3bjleaTqN0oTwQWFVVVwIkWQIsAgYD3CLgje30mcB7k6Sdv6SqbgN+mGRVu79vTE/zJUlSlzzRljaO/4c03VJVd75CciSwsKr+vH3+POCgqjpuYJ3vteusbp9fARxEE+rOr6r/bOd/APhcVZ05VONY4Nj26QOByzbweHYGrtvAbTeGda1r3X7WnU3Hal3rWrefNa1rXevOzrp7VtXc8RZsFoOYVNUpwCkbu58kK6pqwTQ0ybrWte4sqDubjtW61rVuP2ta17rWte6wUQYxWQPsPvB8t3beuOsk2QrYHrh+xG0lSZIkSSMYJcAtB+Yn2SvJ1jSDkiwdWmcpcHQ7fSTwxWr6Zi4FFrejVO4FzAe+OT1NlyRJkqTZZdIulFW1LslxwNnAHODUqlqZ5ERgRVUtBT4AfKQdpOQGmpBHu94ZNAOerANe2vEIlBvdDdO61rXurKo7m47Vuta1bj9rWte61rXueiYdxESSJEmStHkY6Yu8JUmSJEkzzwAnSZIkST1hgJMkSZKkntgsvgduQyV5ELAI2LWdtQZYWlWXzlyrutMe767ABVX1i4H5C6vq8x3WPRCoqlqeZD9gIfD9qlrWVc1x2vAfVfX8TVVvoO4fAQcC36uqczqscxBwaVXdnOTuwPHAI2gGAHpLVf28g5ovBz5VVddO974nqTs2mu2PquoLSZ4L/CFwKXBKVf2mw9p7A8+g+XqT24EfAKdV1c1d1ZQkSZpOvR3EJMlrgOcAS4DV7ezdaE4Ml1TVSTPQphdU1Qc72vfLgZfSnOTuD7yiqj7TLruwqh7RUd0TgMNowv65wEHAecAhwNlV9eYOag5/TUWA/w/4IkBVHTHdNQdqf7OqDmyn/4LmNf8UcCjw2a7eV0lWAg9vR309BbgFOBN4cjv/GR3U/DnwS+AK4GPAx6tq7XTXGafuR2neT/cAbgK2BT5Jc6ypqqMn3nqj6r4ceBrwP8BTgW+39f8EeElVfamLupIkbY6S3KeqfjbT7dgUkuxUVdfPdDumS58D3A+Ahwx/Wt9+ur+yqubPQJuuqao9Otr3xcBjquoXSebRnNx/pKreneTbVXVAh3X3B7YBfgLsNnCV6IKqelgHNS+kufL070DRBLiP8buvp/jydNccqP3b1zLJcuCpVbU2yT2B86vqoR3VvbSqHtxOrxfIk1xUVft3UPPbwCOBg4GjgCOAb9G81p+sqv+d7ppt3e9W1cOSbEVz1XyXqro9SYDvdPGeauteDOzf1roHsKyqnphkD+AzXf0f0uwzm06KYMs7MZqtkmwPvBZ4OnAfmr+/PwM+A5xUVTfNQJs+V1WHdbTv7WiOdzfgc1V12sCy91XVSzqqez/gBOAO4A3Ay4Bn0nxA/4qq+nFHdXccnkXzN/8AmjxwQwc1f9tDrH1/vRN4FPA94JVV9dPprtnWOgn4f1V1XZIFwBk0r/ddged3dR7Znr9+EvhYVV3RRY0xfb4H7g5gl3Hm379d1okk353gcTFw367qAncZ6zZZVVcBTwQOS/JOmv+EXVlXVbdX1S3AFWNdzarqVrp7nRfQ/FJ5HfDz9srIrVX15S7DW+suSXZIshPNL7S1AFX1S5rvMuzK95K8oJ3+TvsLhyT7Al11KayquqOqzqmqF9H8f3ofTRfZKzuqCc1rvDVwL5qrcNu387eh+eXapbFu49vQXPmjqq7psm6S7ZOclOT7SW5Icn2SS9t59+6q7iRt+lyH+94uyVuTfKTtHju47H0d1r1fkn9JcnKSnZK8McnFSc5Icv8O6+449NgJ+Gb7e2T4hGk66y4cmN4+yQfav0WnJensb1H7vt25nV6Q5ErggiRXJ3lCh3UvTPL6JPt0VWOCuguSnJfkP5PsnuTcJD9PsjxJVx+cbpvkxCQr21prk5yf5Jgu6g04A7gReGJV7VhVO9H0frmxXdaJJI+Y4PFImg+Qu/JBmvOnTwCLk3wiyTbtskd3WPdDNB9SX0vTo+lWml4hXwH+tcO619GcW409VtDclnNhO92FtwxM/yPwY+CPgeXA+zuqCXB4VV3XTr8DOKqqHkDTe+wfO6y7A3Bv4Lwk30zyyiTjZZWN1ud74P4a+O8kl9P8JwDYA3gAcFyHde8LPIXmF9qgAF/vsO5Pk+xfVRcBtFfingacCnRyVaj16yT3aAPcI8dmtp+kdBLgquoO4F1JPt7++1M23Xt1e5pfbAEqyf2r6sdJtqXboPznwLuTvJ7ml+w3klxL897+845qrnc87dXspcDS9gpVVz4AfB+YQxPSP96eBD6apkt0V/4dWJ7kAuBxwNsAkswFpv2TxwFn0HT/fWJV/aSteT/g6HbZoV0UTTJRt+rQ/UnR5TQnRS9M8kzguVV1G92fFJ0F3JPmpOijNCdFT6c5KVrUUd3rgKuH5o2dFBWwd0d13wKM3fs8eGL0DJoTo6d3VPfwqjq+nR47MVrefth0Gs0HcF0YPDH6CU1PgdOr6kcd1RvzPpqrJfem+Rv/yqo6JMmT22WP6aDmR2m67j8FeDbNe3oJ8Pok+1bV33VQE2BeVb1tcEb7O+ttSV7YUU1oTua/zPh/Y+/dYd19quqZ7fSnk7wO+GKSzm7TaN23qt4DkOQlA6/5e5K8qMO6f0MTYP6mqi5u6/+wqvbqsOagBQO9id6VpJPbJVpbJdmqqtYBd6+q5QBV9YOBkN6FG6vq1cCrkzyO5lavC5NcSnNVbtq+0Lu3XSgBktyFZoCJwUFMllfV7R3W/ADwwar66jjLTquq546z2XTU3Y3mathPxln22Kr6Wkd1t2lPvIbn7wzcf+yXQJeSHA48tsM/WqO04R40v3R/2HGd7YC9aALr6q66F7S19q2qH3S1/0lq7wJQVT9qr0IdDFxTVd/suO5DgAfTDErz/S5rDdS8rKoeONVl01D3diY+KXp0Vd29o7rrdfltT4qeStNF99wO79cd7P68Xnf24TZNc93/wwycFGWgq/U4r3mXx3sp8ND2ft3zq+rRA8su7rCb+eDxjp0YPYOm29m0nhgN1b2z91Unty8k+U5VPXzg+fKqelR7znNJVT1oumu2dc4BvgB8eOxvT3s19xjgkKo6uKO63wP+pKouH2fZtVW1e0d1L6W5FeeOgXnH0ASdbatqz47q/vbnm+Qfqur1A8s6+z/U7n834F00Hw6fQHPbQlcfMpFkNU23ydCMKbBPtcEj7e0UHdV9Gc0HWicBj6f5AOiTwJOAvavqeR3V/b0xKZLMofkbcVRVvWD8Laeuz1fgxq7UnL+Ja0746UhX4a3d9+o7WdZJeGv3/XvhrZ1/Hc0nz52rqrNoPlmfMe0VyE7DW1vnZuA7Xddpa81IeGtr/2hg+iaaezo3Rd2VwMpNUWvA1Un+lvFPirocAfRS4C8nOinqsO42Se4ydlJUVW9OsoZm8JhtO6w7eEvAfwwtm9NV0ar6xySn03yiPHZStCk+Gb1PklfRnBhtlyRjJ0Z0e3vE+4Blae4x+XySd/O7E6OLOqz7W1X1FeAr7UnaITT38HYS4IBfJTmUpndGJXl6VX06TXfRrj4s/mWSP6qqr7ZXg26A5pwnSZc9QY6iGQH5y+3vqAJ+StMz49kd1n0jE79nX9Zh3c/SvG+/MDajqj7UXuF9T4d1P5Nk26r6xVB4ewBwWYd1x84ln9W+r86luY2hS/9Gc7sEwIeBnYG1bS+Ui7oqWlXvSXNr018B+9LknfnAp4F/6KouzcjWw225naa3xLSOFt/rK3CStLlLsgPNSdEimoEB4HcnRSdV1XB37OmqeyRwcVX93gnB2EloR3XfDpxTVV8Ymr8QeE91NMBUkhOBt9fAV6y08x9A8zof2UXdoVpHAH9H0xXtfh3XOmFo1vuqGXDpfjSvQ2dfu5Lkiax/YnQtzYnRqW2XpS5qLqmqxV3se5K6DwfeTnPLwCtpjvtomh4/f1FV037rRJKH0XT5nk/zgdML265fc4HnVNU/T3fNgdoPohnU4/zatF9XNFNfkzRR3cOqqst7hWf8eGk+gNinqr7XZd3N4Vi3yLpV5cOHDx8+ZuABvMC6W1Zd4O7AH8yW47XullMTeDnNFaBPA1cBiwaWXbgF1n3ZLKu7yV/n2XSsm/p4vQInSTNk+J4a61rXutadqZqZ2a8rsu4WWHc2Heumrtvre+AkaXOX5LsTLaLDrx6xrnWt28+6M3WsDH1dUdtV9swkezL+YEjWte7mWHNW1DXASVK3ZuqrR6xrXev2s+5s+7oi6265dWfTsW7SugY4SerWf9EMSX3R8IIkX7Kuda1r3c2gJsDzgfUGoalmUJrnJ+nyS5etu+XWnU3Huknreg+cJEmSJPVEl98VI0mSJEmaRgY4SZIkSeoJA5wkSZIk9YQBTpIkSZJ64v8HmROfyPt4KoQAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 1080x936 with 3 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig = plt.figure(figsize=(15, 13))\n",
    "\n",
    "ax1 = plt.subplot(3, 1, 1)\n",
    "train[range(n_labels)].mean(axis=0).plot(kind=\"bar\", ax=ax1, title=\"distribution of labels in train\")\n",
    "\n",
    "ax2 = plt.subplot(3, 1, 2)\n",
    "valid[range(n_labels)].mean(axis=0).plot(kind=\"bar\", ax=ax2, title=\"distribution of labels in validation\")\n",
    "\n",
    "ax3 = plt.subplot(3, 1, 3)\n",
    "test[range(n_labels)].mean(axis=0).plot(kind=\"bar\", ax=ax3, title=\"distribution of labels in test\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "ac2ef7fc",
   "metadata": {},
   "outputs": [],
   "source": [
    "def inspect_data(label, n=5):\n",
    "    samples = train[train[label] == 1].sample(5)\n",
    "    sentiment = mapping[label]\n",
    "    \n",
    "    print(f\"examples from {sentiment}\\n\")\n",
    "    for text in samples[\"text\"]:\n",
    "        print(text + \"\\n---\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "df55477f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "examples from caring\n",
      "\n",
      "Just be sure to take some condoms with you.\n",
      "---\n",
      "If he isn't respecting your clear boundaries then you should block him and move on with your life. He's not worth your time.\n",
      "---\n",
      "Don't worry dude. The ladies could never see it if you remain to be a virgin and keep your pants up.\n",
      "---\n",
      "Are you okay? What's wrong?\n",
      "---\n",
      "Try Rescue Melatonin Strawberry gummies and Natural Calm in hot water. They saved my life.\n",
      "---\n"
     ]
    }
   ],
   "source": [
    "inspect_data(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "20f27620",
   "metadata": {},
   "source": [
    "Define a Pytorch dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "16312f0d",
   "metadata": {},
   "outputs": [],
   "source": [
    "class GoEmotionDataset():\n",
    "    def __init__(self, texts, targets):\n",
    "        self.texts = texts\n",
    "        self.targets = targets\n",
    "        self.tokenizer = transformers.SqueezeBertTokenizer.from_pretrained(\n",
    "            \"squeezebert/squeezebert-uncased\", do_lower_case=True\n",
    "        )\n",
    "        self.max_len = 35\n",
    "        \n",
    "    def __len__(self):\n",
    "        return len(self.texts)\n",
    "    \n",
    "    def __getitem__(self, index):\n",
    "        target = self.targets[index]\n",
    "        text = self.texts[index]\n",
    "        \n",
    "        inputs = self.tokenizer.encode_plus(text, \n",
    "                                            None, \n",
    "                                            add_special_tokens=True, \n",
    "                                            max_length=self.max_len,\n",
    "                                            padding=\"max_length\",\n",
    "                                            truncation=True)\n",
    "        \n",
    "        ids = inputs[\"input_ids\"]\n",
    "        mask = inputs[\"attention_mask\"]\n",
    "        \n",
    "        return {\n",
    "            \"ids\": torch.tensor(ids, dtype=torch.long),\n",
    "            \"mask\": torch.tensor(mask, dtype=torch.long),\n",
    "            \"targets\": torch.tensor(self.targets[index], dtype=torch.long),\n",
    "        }"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "24df258c",
   "metadata": {},
   "source": [
    "define a tez model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "709da1da",
   "metadata": {},
   "outputs": [],
   "source": [
    "class config:\n",
    "    def __init__(self):\n",
    "        self.val_strategy = \"batch\"\n",
    "\n",
    "class EmotionClassifier(tez.Model):\n",
    "    def __init__(self, num_train_steps, num_classes):\n",
    "        super().__init__()\n",
    "        self.bert = transformers.SqueezeBertModel.from_pretrained(\"squeezebert/squeezebert-uncased\")\n",
    "        self.bert_drop = nn.Dropout(0.3)\n",
    "        self.out = nn.Linear(768, num_classes)\n",
    "        self.num_train_steps = num_train_steps\n",
    "        self.config = config()\n",
    "        \n",
    "    def fetch_optimizer(self):\n",
    "        param_optimizer = list(self.named_parameters())\n",
    "        no_decay = [\"bias\", \"LayerNorm.bias\"]\n",
    "        optimizer_parameters = [\n",
    "            {\n",
    "                \"params\": [\n",
    "                    p for n, p in param_optimizer if not any(nd in n for nd in no_decay)\n",
    "                ],\n",
    "                \"weight_decay\": 0.001,\n",
    "            },\n",
    "            {\n",
    "                \"params\": [\n",
    "                    p for n, p in param_optimizer if any(nd in n for nd in no_decay)\n",
    "                ],\n",
    "                \"weight_decay\": 0.0,\n",
    "            }\n",
    "        ]\n",
    "        opt = AdamW(optimizer_parameters, lr=3e-5)\n",
    "        return opt\n",
    "    \n",
    "    def fetch_scheduler(self):\n",
    "        sch = get_linear_schedule_with_warmup(\n",
    "            self.optimizer, num_warmup_steps=0, num_training_steps=self.num_train_steps\n",
    "        )\n",
    "        return sch\n",
    "    \n",
    "    def loss(self, outputs, targets):\n",
    "        if targets is None:\n",
    "            return None\n",
    "        return nn.BCEWithLogitsLoss()(outputs, targets.float())\n",
    "    \n",
    "    def monitor_metrics(self, outputs, targets):\n",
    "        if targets is None:\n",
    "            return {}\n",
    "        \n",
    "        outputs = torch.sigmoid(outputs)\n",
    "        outputs = outputs.cpu().detach().numpy()\n",
    "        targets = targets.cpu().detach().numpy()\n",
    "        \n",
    "        fpr_micro, tpr_micro, _ = metrics.roc_curve(targets.ravel(), outputs.ravel())\n",
    "        auc_micro = metrics.auc(fpr_micro, tpr_micro)\n",
    "        return {\"auc\": auc_micro}\n",
    "    \n",
    "    def forward(self, ids, mask, targets= None):\n",
    "        o_2 = self.bert(ids, attention_mask=mask)[\"pooler_output\"]\n",
    "        b_o = self.bert_drop(o_2)\n",
    "        output = self.out(b_o)\n",
    "        loss = self.loss(output, targets)\n",
    "        acc = self.monitor_metrics(output, targets)\n",
    "        return output, loss, acc"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2161a864",
   "metadata": {},
   "source": [
    "Star the training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "a80f65a5",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dataset = GoEmotionDataset(train.text.tolist(), train[range(n_labels)].values.tolist())\n",
    "valid_dataset = GoEmotionDataset(valid.text.tolist(), valid[range(n_labels)].values.tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "77462b4f",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at squeezebert/squeezebert-uncased were not used when initializing SqueezeBertModel: ['cls.predictions.decoder.weight', 'cls.predictions.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.dense.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.dense.bias', 'cls.seq_relationship.weight', 'cls.predictions.transform.LayerNorm.bias']\n",
      "- This IS expected if you are initializing SqueezeBertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing SqueezeBertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    }
   ],
   "source": [
    "n_train_steps = int(len(train) / 32 * 10)\n",
    "model = EmotionClassifier(n_train_steps, n_labels)\n",
    "\n",
    "tb_logger = tez.callbacks.TensorBoardLogger(log_dir=\"logs/\")\n",
    "es = tez.callbacks.EarlyStopping(monitor=\"valid_loss\", model_path=\"export/model.bin\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dc2e5496",
   "metadata": {},
   "outputs": [],
   "source": [
    "# model.fit(train_dataset, \n",
    "#           valid_dataset, \n",
    "#           train_bs=64, \n",
    "#           device=\"cpu\", \n",
    "#           epochs=8, \n",
    "#           callbacks=[tb_logger, es], \n",
    "#           fp16=True, \n",
    "#           n_jobs=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "021bb5a2",
   "metadata": {},
   "outputs": [],
   "source": [
    "model.load(\"export/model.bin\", device=\"cpu\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "201a1f83",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_dataset = GoEmotionDataset(test.text.tolist(), test[range(n_labels)].values.tolist())\n",
    "dataloader = DataLoader(test_dataset, batch_size=64, shuffle=False, drop_last=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cc654844",
   "metadata": {},
   "source": [
    "Evaluate on test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "101c61f3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4b4f01f58fa0452f90220b2bde4dc045",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/85 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "outputs = []\n",
    "\n",
    "with torch.no_grad():\n",
    "    for i, batch in tqdm(enumerate(dataloader), total=len(dataloader)):\n",
    "        output, loss, acc = model.forward(batch[\"ids\"].to(\"cpu\"),\n",
    "                                        batch[\"mask\"].to(\"cpu\"),\n",
    "                                        batch[\"targets\"].to(\"cpu\")\n",
    "                                        )\n",
    "        outputs.append(output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "92ad4fe5",
   "metadata": {},
   "outputs": [],
   "source": [
    "outputs = torch.cat(outputs)\n",
    "outputs = torch.sigmoid(outputs)\n",
    "outputs = outputs.cpu().detach().numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "7f0bbb81",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:title={'center':'roc auc score per class on test data'}>"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABIEAAAFBCAYAAADg5r/SAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAp10lEQVR4nO3de7xtdV0v/M8XyCuKJra9gGKCFUWZ7LxkF3jUAlHpJHk7aVpKnaIrWpQ+5eNRH7RjPV5LT2pl6Ra7GAF5S7x0SgNMRUQNCQNUFFQQtBT5Pn+MsWmyXGuvxZqLPed2vN+v13rtueYYa8zPnHOttcf6zN/vN6u7AwAAAMA3tr0WHQAAAACAm54SCAAAAGAClEAAAAAAE6AEAgAAAJgAJRAAAADABCiBAAAAACZACQQAMKeqemZV/dmic+xpquqiqnrwonMAwFQogQAAJqqqjqiqS7boWO+oqidvxbHWOH5X1cE31fEBYAqUQACwJKpqn0VnWCaLfDw8FwDANyIlEAAs0Dgd5jeq6oNJrqmqfarqEVV1XlV9YRxd8R0z+x9YVX9VVZ+tqiuq6iVrHPe+VfVP4zE+VVUvqaqbjdsOGkdV7DOz/w1GcVTVU6rq/Kr6YlV9uKrus8ptVFX9flV9pqquqqpzq+q7xm23rKoXVNUnqurKqvqHqrrluG1X92+1x+P+VfWP4/4fqKoj1nk8f3PM/PmqenVV3WJm+8Oq6v3jsf6xqr57V7e9yvG/s6reWlWfq6rLquq31sjxhqr69Hjf31VV3zmz7aFjvi9W1aVV9dTx+v2r6rQx2+eq6t1Vteq5WlV9f1WdNR7/rKr6/plt76iq/1lV/2e8jbdU1f6rHOPWSf4uyV2q6urx4y5VtVdVnVRVHx+/x06pqm8ev+YWVfVn4/VfGG97W1U9J8kPJnnJeJy1vi8fP35PXFFVT1+xbVffs+8ad/vAePxHV9Xtx8frs+NzfVpVHbDa7QIAAyUQACzeY5Mck+R2Sb41yeuS/EqSOyY5I8nfVtXNqmrvJKcl+USSg5LcNcmONY75tSS/mmT/JA9I8qAkP7+RMFX1E0memeQJSW6b5BFJrlhl1x9J8kNJ7pVkvySPmtnvfyU5PMn3J/nmJL+e5Lqqutda92/muLOPx7Ykpyd59nicpyb5y6q64y7uwn9P8qNJ7jlme8Z4v743yauS/GySOyR5eZJTq+rmq912d1+74nG5TZK3JXlTkrskOTjJ36+R4e+SHJLkW5K8L8mfz2x7ZZKf7e7bJPmuJG8frz8xySXj47ItyW8l6ZUHHguZ05O8aLwfv5fk9Kq6w8xuj0vypPH2b5bhcbuB7r4mydFJPtnd+44fn0zyi0l+LMkPj/fz80leOn7ZT2V4rg8cb/vnkny5u5+e5N1JThiPc8IquQ9N8gdJHj8e9w5JZkubNb9nu/uHxn2+Zzz+6zOcx746yd2T3C3Jl5OsWj4BAAMlEAAs3ou6++Lu/nKSRyc5vbvf2t1fzVCm3DJDmXLfDH88P627r+nu/+juf1jtgN19Tne/p7uv7e6LMhQeP7zBPE9O8vzuPqsHF3T3J1bZ76tJbpPk25NUd5/f3Z8aR6/8dJJf7u5Lu/tr3f2P3f2f69y/1R6Pn0xyRnef0d3Xdfdbk5yd5KG7yP+S8es/l+Q5GYqdJDk+ycu7+71jpj9J8p9J7r/Gba/0sCSf7u4XjI/9F7v7vasF6O5Xjdv/M0Oh9j1Vtd/M43ZoVd22uz/f3e+buf7OSe7e3V/t7nd399eVQBlKqn/t7teMz+/rknwkycNn9nl1d39svB+nJLn3Lh6vlX4uydO7+5KZ/MeNI6O+mqG8OXh8DM/p7qs2eNzjkpzW3e8aj/t/J7lu58Yb+z3b3Vd0919295e6+4sZnuuNfo8DwCQpgQBg8S6euXyXDCN9kiTdfd24/a4ZRl98YuUIldVU1b3G6TGfrqqrkjw3wwiLjTgwycfX26m7355h5MVLk3ymql5RVbcdb+cWaxxjV/dvp9nH4+5JfmKcIvSFqvpCkh/IUJasZfbrPzHe5s5jnbjiWAfObF/5tStt6HGpqr2r6uRxOtVVSS4aN+18/B+ZocT6RFW9s6oeMF7/u0kuSPKWqrqwqk5a4yZu8BiOPpEbPoafnrn8pST7rpd7xt2T/PXMY3R+hlE625K8Jsmbk+yoqk9W1fOr6ps2eNy7ZObxHUciXT/C7MZ+z1bVrarq5eP0squSvCvJ7cYRcwDAKpRAALB4s6M9Ppnhj/Akw7o7GcqHSzP8AX232tiixX+QYXTIId192wxTi2rcds34761m9r/TzOWLM0ylWj9494u6+/Akh2aYevW0JJcn+Y81jrGr+3f9YVdkeU13327m49bdffIuYh04c/lu423uPNZzVhzrVuNImtVue6WLM0zXW8/jkhyb5MEZpk4dNF5fSTKOsDo2w1StN2YYqZNx5NCJ3f2tGabg/VpVPWiV49/gMZy5n5eusu96Vru/Fyc5esXjdItxVNdXu/v/6e5DM4zeeliGaYNrHWvWpzLz3FTVrTKMKtppV9+zqzkxybclud+4/84pY7v6GgCYNCUQACyXU5IcU1UPGkdYnJhhytI/JvnnDH9In1xVtx4X6X3gGse5TZKrklxdVd+e5H/s3NDdn81QGPzkOGrlp3PDwuaPkjy1qg6vwcFVtbJ0SFV9X1Xdb8x5TYbi57pxdM+rkvzeuNDw3lX1gHHtnV3dv9X8WZKHV9WPjse5RQ1va76rBYB/oaoOGNfOeXqS14/X/+8kPzdmrvExPGZc62cjTkty56r6laq6eVXdpqrut8p+txnv0xUZirbn7twwru3036tqv3E63FUZp0TVsGj1wWMxdmWG0TfXfd3Rh3WU7lVVj6th4exHZyjhTtvg/Zh1WZI7zExVS5I/TPKcnc95Vd2xqo4dLx9ZVYeNo22uyjA97LqZY+2qJPuLJA+rqh8Y14B6Vm54Lrrm9+wax79NhnWAvjA+17+z0TsNAFOlBAKAJdLdH82wDs6LM4yoeXiSh3f3V7r7a+PnByf59wyLCD96jUM9NcOIlC9mKD9ev2L7UzKM2rkiyXdmpoTp7jdkWF/ltePXvzHDoswr3XY89uczTEe6IsOUpp23f26Ss5J8Lsnzkuy1q/u3xuNxcYZRNb+V5LMZRqk8Lbs+h3ltkrckuTDD9K1nj8c6e7zfLxkzX5Dkibs4zsosX0zykDHzp5P8a5IjV9n1TzM8Hpcm+XCS96zY/vgkF41TmH4uw0LWybCQ9NuSXJ3kn5K8rLvPXCXHFRlG4JyY4TH/9SQP6+7LN3pfZo71kQwLdV84Tv+6S5IXJjk1w7S0L475d5Zdd8pQ5lyVYZrYOzNMEcv4dcfV8E5dL1rlts5L8gsZnp9PZXgOLpnZZb3v2Wcm+ZMx56OS/H8Z1pO6fMz4pht7/wFgamr19QYBAPY8VXVRkid399sWnQUAYNkYCQQAAAAwAUogAAAAgAkwHQwAAABgAowEAgAAAJiAfRZ1w/vvv38fdNBBW3a8a665Jre+9a237HhbTb75LHO+Zc6WyDcv+eYj3+Ytc7ZEvnnJNx/5Nm+ZsyXyzUu++ci3ecucLZlevnPOOefy7r7jqhu7eyEfhx9+eG+lM888c0uPt9Xkm88y51vmbN3yzUu++ci3ecucrVu+eck3H/k2b5mzdcs3L/nmI9/mLXO27unlS3J2r9HFmA4GAAAAMAFKIAAAAIAJUAIBAAAATIASCAAAAGAClEAAAAAAE6AEAgAAAJiAdUugqnpVVX2mqj60xvaqqhdV1QVV9cGqus/WxwQAAABgHhsZCfTHSY7axfajkxwyfhyf5A/mjwUAAADAVlq3BOrudyX53C52OTbJn/bgPUluV1V33qqAAAAAAMyvunv9naoOSnJad3/XKttOS3Jyd//D+PnfJ/mN7j57lX2PzzBaKNu2bTt8x44d86WfcfXVV2fffffdsuNtNfnms8z5ljlbIt+85JuPfJu3zNkS+eYl33zk27xlzpbINy/55iPf5i1ztmR6+Y488shzunv7qhu7e92PJAcl+dAa205L8gMzn/99ku3rHfPwww/vrXTmmWdu6fG2mnzzWeZ8y5ytW755yTcf+TZvmbN1yzcv+eYj3+Ytc7Zu+eYl33zk27xlztY9vXxJzu41uph9tqBkujTJgTOfHzBeBwDAEjnopNM3tN+Jh12bJ25g34tOPmbeSADAbrQVbxF/apInjO8Sdv8kV3b3p7bguAAAAABskXVHAlXV65IckWT/qrokye8k+aYk6e4/THJGkocmuSDJl5I86aYKCwAAAMDmrFsCdfdj19neSX5hyxIBAAAAsOW2Yk0gAABggaz3BMBGKIHm5D9cAAAANsvflOxOW7EwNAAAAABLTgkEAAAAMAFKIAAAAIAJUAIBAAAATICFoQEAtojFPQGAZWYkEAAAAMAELP1IIK+oATcFv1sAAGDXnDN/4zESCAAAAGACln4kELBn8qoBAADAclECAQDABniBA4A9nRIIAHYjf0QCALAo1gQCAAAAmAAlEAAAAMAEKIEAAAAAJsCaQAAAAMAex1qLN56RQAAAAAATYCTQNzjNKOyZ/OwCAABbTQkEAADcpJb9xY1lzwewVZRAsIdysgIAAMCNYU0gAAAAgAkwEggA2GMYBQkAsHlGAgEAAABMgBIIAAAAYAKUQAAAAAAToAQCAAAAmAAlEAAAAMAEeHcwAL7heAcpAAD4ekYCAQAAAEyAEggAAABgApRAAAAAABOgBAIAAACYACUQAAAAwAR4dzAAAACALbaM71hrJBAAAADABBgJxEItYzMKMGV+L7NIvv+Am4LfLfBflEAA3GhOpgAAYM9jOhgAAADABCiBAAAAACbAdDBYg+kuAAAAfCMxEggAAABgApRAAAAAABOwoRKoqo6qqo9W1QVVddIq2+9WVWdW1b9U1Qer6qFbHxUAAACAzVq3BKqqvZO8NMnRSQ5N8tiqOnTFbs9Ickp3f2+SxyR52VYHBQAAAGDzNjIS6L5JLujuC7v7K0l2JDl2xT6d5Lbj5f2SfHLrIgIAAAAwr+ruXe9QdVySo7r7yePnj09yv+4+YWafOyd5S5LbJ7l1kgd39zmrHOv4JMcnybZt2w7fsWPHugHPvfTKDd2RbbdMLvvy+vsddtf9NnS8jZJvPsucb5mzJfLNS775yLd5y5wtkW9e8s1Hvs1b5myJfLvL1VdfnX333Xcht70Ri8q37M/vMudb5myJfGs58sgjz+nu7att26oS6NfGY72gqh6Q5JVJvqu7r1vruNu3b++zzz573fA35m26X3Du+u94v9Vv0y3ffJY53zJnS+Sbl3zzkW/zljlbIt+85JuPfJu3zNkS+XaXd7zjHTniiCMWctsbsah8y/78LnO+Zc6WyLeWqlqzBNrIdLBLkxw48/kB43WzfibJKUnS3f+U5BZJ9t9QOgAAAABuchspgc5KckhV3aOqbpZh4edTV+zz70kelCRV9R0ZSqDPbmVQAAAAADZv3RKou69NckKSNyc5P8O7gJ1XVc+qqkeMu52Y5ClV9YEkr0vyxF5vnhkAAAAAu836k86SdPcZSc5Ycd1vz1z+cJIHbm00AAAAALbKRqaDAQAAALCHUwIBAAAATIASCAAAAGACNrQmEAAAAItx0Emnb2i/Ew+7Nk/cwL4XnXzMvJGAPZSRQAAAAAAToAQCAAAAmAAlEAAAAMAEKIEAAAAAJkAJBAAAADABSiAAAACACVACAQAAAEyAEggAAABgAvZZdAAAAAD2XAeddPqG9jvxsGvzxA3se9HJx8wbCViDkUAAAAAAE6AEAgAAAJgAJRAAAADABCiBAAAAACZACQQAAAAwAUogAAAAgAlQAgEAAABMgBIIAAAAYAKUQAAAAAAToAQCAAAAmAAlEAAAAMAEKIEAAAAAJkAJBAAAADABSiAAAACACVACAQAAAEyAEggAAABgApRAAAAAABOgBAIAAACYACUQAAAAwAQogQAAAAAmQAkEAAAAMAFKIAAAAIAJUAIBAAAATIASCAAAAGAClEAAAAAAE6AEAgAAAJgAJRAAAADABCiBAAAAACZACQQAAAAwAUogAAAAgAlQAgEAAABMwIZKoKo6qqo+WlUXVNVJa+zzqKr6cFWdV1Wv3dqYAAAAAMxjn/V2qKq9k7w0yUOSXJLkrKo6tbs/PLPPIUl+M8kDu/vzVfUtN1VgAAAAAG68jYwEum+SC7r7wu7+SpIdSY5dsc9Tkry0uz+fJN39ma2NCQAAAMA8qrt3vUPVcUmO6u4nj58/Psn9uvuEmX3emORjSR6YZO8kz+zuN61yrOOTHJ8k27ZtO3zHjh3rBjz30is3dEe23TK57Mvr73fYXffb0PE2Sr75LHO+Zc6WyDcv+eYj3+Ytc7ZEvnnJNx/5Nm+ZsyXyzUu++ci3ecucLZFvLUceeeQ53b19tW1bVQKdluSrSR6V5IAk70pyWHd/Ya3jbt++vc8+++x1wx900unr7pMkJx52bV5w7rqz23LRycds6HgbJd98ljnfMmdL5JuXfPORb/OWOVsi37zkm498m7fM2RL55iXffOTbvGXOlsi3lqpaswTayHSwS5McOPP5AeN1sy5Jcmp3f7W7/y3DqKBDNpQOAAAAgJvcRkqgs5IcUlX3qKqbJXlMklNX7PPGJEckSVXtn+ReSS7cupgAAAAAzGPdEqi7r01yQpI3Jzk/ySndfV5VPauqHjHu9uYkV1TVh5OcmeRp3X3FTRUaAAAAgBtn/UlnSbr7jCRnrLjut2cud5JfGz8AAAAAWDIbmQ4GAAAAwB5OCQQAAAAwAUogAAAAgAlQAgEAAABMgBIIAAAAYAKUQAAAAAAToAQCAAAAmAAlEAAAAMAEKIEAAAAAJkAJBAAAADABSiAAAACACVACAQAAAEyAEggAAABgApRAAAAAABOgBAIAAACYACUQAAAAwAQogQAAAAAmQAkEAAAAMAFKIAAAAIAJUAIBAAAATIASCAAAAGAClEAAAAAAE6AEAgAAAJgAJRAAAADABCiBAAAAACZACQQAAAAwAUogAAAAgAlQAgEAAABMgBIIAAAAYAKUQAAAAAAToAQCAAAAmAAlEAAAAMAEKIEAAAAAJkAJBAAAADABSiAAAACACVACAQAAAEyAEggAAABgApRAAAAAABOgBAIAAACYACUQAAAAwAQogQAAAAAmQAkEAAAAMAFKIAAAAIAJ2FAJVFVHVdVHq+qCqjppF/s9sqq6qrZvXUQAAAAA5rVuCVRVeyd5aZKjkxya5LFVdegq+90myS8nee9WhwQAAABgPhsZCXTfJBd094Xd/ZUkO5Icu8p+/zPJ85L8xxbmAwAAAGALVHfveoeq45Ic1d1PHj9/fJL7dfcJM/vcJ8nTu/uRVfWOJE/t7rNXOdbxSY5Pkm3bth2+Y8eOdQOee+mVG7oj226ZXPbl9fc77K77beh4GyXffJY53zJnS+Sbl3zzkW/zljlbIt+85JuPfJu3zNkS+eYl33zk27xlzpbIt5YjjzzynO5edZmeuUugqtoryduTPLG7L9pVCTRr+/btffbZu9wlSXLQSaevu0+SnHjYtXnBufusu99FJx+zoeNtlHzzWeZ8y5wtkW9e8s1Hvs1b5myJfPOSbz7ybd4yZ0vkm5d885Fv85Y5WyLfWqpqzRJoI9PBLk1y4MznB4zX7XSbJN+V5B1VdVGS+yc51eLQAAAAAMtjIyXQWUkOqap7VNXNkjwmyak7N3b3ld29f3cf1N0HJXlPkkesNxIIAAAAgN1n3RKou69NckKSNyc5P8kp3X1eVT2rqh5xUwcEAAAAYH7rTzpL0t1nJDljxXW/vca+R8wfCwAAAICttJHpYAAAAADs4ZRAAAAAABOgBAIAAACYACUQAAAAwAQogQAAAAAmQAkEAAAAMAFKIAAAAIAJUAIBAAAATIASCAAAAGAClEAAAAAAE6AEAgAAAJgAJRAAAADABCiBAAAAACZACQQAAAAwAUogAAAAgAlQAgEAAABMgBIIAAAAYAKUQAAAAAAToAQCAAAAmAAlEAAAAMAEKIEAAAAAJkAJBAAAADABSiAAAACACVACAQAAAEyAEggAAABgApRAAAAAABOgBAIAAACYACUQAAAAwAQogQAAAAAmQAkEAAAAMAFKIAAAAIAJUAIBAAAATIASCAAAAGAClEAAAAAAE6AEAgAAAJgAJRAAAADABCiBAAAAACZACQQAAAAwAUogAAAAgAlQAgEAAABMgBIIAAAAYAKUQAAAAAAToAQCAAAAmIANlUBVdVRVfbSqLqiqk1bZ/mtV9eGq+mBV/X1V3X3rowIAAACwWeuWQFW1d5KXJjk6yaFJHltVh67Y7V+SbO/u707yF0mev9VBAQAAANi8jYwEum+SC7r7wu7+SpIdSY6d3aG7z+zuL42fvifJAVsbEwAAAIB5VHfveoeq45Ic1d1PHj9/fJL7dfcJa+z/kiSf7u5nr7Lt+CTHJ8m2bdsO37Fjx7oBz730ynX3SZJtt0wu+/L6+x121/02dLyNkm8+y5xvmbMl8s1LvvnIt3nLnC2Rb17yzUe+zVvmbIl885JvPvJt3jJnS+Rby5FHHnlOd29fbduWlkBV9ZNJTkjyw939n7s67vbt2/vss89eN/xBJ52+7j5JcuJh1+YF5+6z7n4XnXzMho63UfLNZ5nzLXO2RL55yTcf+TZvmbMl8s1LvvnIt3nLnC2Rb17yzUe+zVvmbIl8a6mqNUug9W8luTTJgTOfHzBet/JGHpzk6dlAAQQAAADA7rWRNYHOSnJIVd2jqm6W5DFJTp3doaq+N8nLkzyiuz+z9TEBAAAAmMe6JVB3X5thitebk5yf5JTuPq+qnlVVjxh3+90k+yZ5Q1W9v6pOXeNwAAAAACzARqaDpbvPSHLGiut+e+byg7c4FwAAAABbaCPTwQAAAADYwymBAAAAACZACQQAAAAwAUogAAAAgAlQAgEAAABMgBIIAAAAYAKUQAAAAAAToAQCAAAAmAAlEAAAAMAEKIEAAAAAJkAJBAAAADABSiAAAACACVACAQAAAEyAEggAAABgApRAAAAAABOgBAIAAACYACUQAAAAwAQogQAAAAAmQAkEAAAAMAFKIAAAAIAJUAIBAAAATIASCAAAAGAClEAAAAAAE6AEAgAAAJgAJRAAAADABCiBAAAAACZACQQAAAAwAUogAAAAgAlQAgEAAABMgBIIAAAAYAKUQAAAAAAToAQCAAAAmAAlEAAAAMAEKIEAAAAAJkAJBAAAADABSiAAAACACVACAQAAAEyAEggAAABgApRAAAAAABOgBAIAAACYACUQAAAAwAQogQAAAAAmQAkEAAAAMAEbKoGq6qiq+mhVXVBVJ62y/eZV9fpx+3ur6qAtTwoAAADApq1bAlXV3klemuToJIcmeWxVHbpit59J8vnuPjjJ7yd53lYHBQAAAGDzNjIS6L5JLujuC7v7K0l2JDl2xT7HJvmT8fJfJHlQVdXWxQQAAABgHtXdu96h6rgkR3X3k8fPH5/kft19wsw+Hxr3uWT8/OPjPpevONbxSY4fP/22JB/dqjuSZP8kl6+71+LIN59lzrfM2RL55iXffOTbvGXOlsg3L/nmI9/mLXO2RL55yTcf+TZvmbMl08t39+6+42ob9tnCG1lXd78iyStuimNX1dndvf2mOPZWkG8+y5xvmbMl8s1LvvnIt3nLnC2Rb17yzUe+zVvmbIl885JvPvJt3jJnS+SbtZHpYJcmOXDm8wPG61bdp6r2SbJfkiu2IiAAAAAA89tICXRWkkOq6h5VdbMkj0ly6op9Tk3yU+Pl45K8vdebZwYAAADAbrPudLDuvraqTkjy5iR7J3lVd59XVc9KcnZ3n5rklUleU1UXJPlchqJod7tJppltIfnms8z5ljlbIt+85JuPfJu3zNkS+eYl33zk27xlzpbINy/55iPf5i1ztkS+6627MDQAAAAAe76NTAcDAAAAYA+nBAIAAACYACUQAAAAwASsuzD0sqqqb09ybJK7jlddmuTU7j5/can2DONjd9ck7+3uq2euP6q737S4ZNfnuG+S7u6zqurQJEcl+Uh3n7HgaKuqqj/t7icsOsdqquoHktw3yYe6+y1LkOd+Sc7v7quq6pZJTkpynyQfTvLc7r5ywfl+Kclfd/fFi8yxmpl3Z/xkd7+tqh6X5PuTnJ/kFd391YUGTFJV35rkx5McmORrST6W5LXdfdVCgwEAAEn20IWhq+o3kjw2yY4kl4xXH5DhD6Qd3X3yorKtp6qe1N2vXuDt/1KSX8jwh+O9k/xyd//NuO193X2fRWUbM/xOkqMzFJRvTXK/JGcmeUiSN3f3cxYYL1V16sqrkhyZ5O1J0t2P2O2hZsNU/XN333e8/JQMz/VfJ/mRJH+76J+NqjovyfeM7zr4iiRfSvIXSR40Xv/jC853ZZJrknw8yeuSvKG7P7vITDtV1Z9n+Lm4VZIvJNk3yV9leOyqu39qcemu/93ysCTvSvLQJP+SIed/S/Lz3f2OhYUDANjDVdW3dPdnFp1jT1VVd+juKxadYxnsqSXQx5J858pXvsdXys/r7kMWk2x9VfXv3X23Bd7+uUke0N1XV9VBGf4Af013v7Cq/qW7v3dR2Wby3TvJzZN8OskBM6NG3tvd373gfO/LMGrlj5J0hhLodRkKyHT3OxeXLpl9DqvqrCQP7e7PVtWtk7ynuw9bcL7zu/s7xss3KB2r6v3dfe+FhRsy/EuSw5M8OMmjkzwiyTkZnuO/6u4vLjDbB7v7u6tqnwwjH+/S3V+rqkrygSX42Tg3yb3HTLdKckZ3H1FVd0vyN4v+3QLLxsn8fJzMsztU1X5JfjPJjyX5lgznfp9J8jdJTu7uLyws3Dqq6u+6++gFZ7hthsfvgCR/192vndn2su7++QVmu1OS30lyXZLfTvKLSR6Z4YXyX+7uTy0qW5JU1TevvCrDOen3Zvgb/nO7P9VMmJkZJOPPye8l+b4kH0ryq9192YLznZzkf3X35VW1PckpGZ7rb0ryhCX4m+19GV7MfV13f3x33/6euibQdUnussr1dx63LVRVfXCNj3OTbFtwvL12TgHr7ouSHJHk6Kr6vQy/XBbt2u7+Wnd/KcnHd04j6e4vZwme2yTbM/wCfnqSK8fRDV/u7ncu+pfJaK+qun1V3SHDfxCfTZLuvibJtYuNliT5UFU9abz8gfGXcqrqXkkWPp0pwzTE67r7Ld39Mxl+z7wsw5TECxcbLXuNRfdtMowG2m+8/uYZ/kNbBjunGN88w0ildPe/Z0nyVdV+VXVyVX2kqj5XVVdU1fnjdbdbdL5dqaq/W/Dt37aq/t+qes04FXF228sWlWsmw52q6g+q6qVVdYeqemZVnVtVp1TVnZcg3zev+LhDkn8ef1+vPNFfRL6jZi7vV1WvHM9bXltViz5vyfgzuv94eXtVXZjkvVX1iar64QXHS1W9r6qeUVX3XHSW1YyP2ZlV9WdVdWBVvbWqrqyqs6pq0S/+7VtVz6qq88ZMn62q91TVExeZa8YpST6f5Iju/ubuvkOGEeCfH7ctVFXdZ42PwzO8qLpor87w98VfJnlMVf1lVd183Hb/xcVKkvxxhhd2L84w6+DLGUYyvzvJHy4u1vUuz/A3x86PszMs5/G+8fKiPXfm8guSfCrJw5OcleTlC0l0Q8d09+Xj5d9N8ujuPjjD7JIXLC7W9W6f5HZJzqyqf66qX62q1fqNm8SeuibQryT5+6r61ww/uElytyQHJzlhUaFmbEvyoxn+g5hVSf5x98e5gcuq6t7d/f4kGUcEPSzJq5IsdJTI6CtVdauxBDp855Vjw7zwEqi7r0vy+1X1hvHfy7JcP0f7ZfiPopJ0Vd25uz9VVftmOUq+Jyd5YVU9I8N/bv9UVRdn+Dl+8kKTDW7wGI2jDU9Ncuo4umWRXpnkI0n2zlBCvmH8Q+j+GabGLtofJTmrqt6b5AeTPC9JquqOSRb6atWMUzJM3Tyiuz+dXP9K4E+N235kgdlSVWtNx60s/mT+1Un+NcOJ/E9X1SOTPK67/zOLP5FPhpP505PcOsPJ/J9nOJn/sQwn88cuKtjo8iSfWHHdzpP5TvKtuz3RDT03yc41AWdP5n88w8n8jy0m1vWO6e6Txss7T+bPGl9AeG2GF2gWafZk/tMZRo++vrs/udBU/+VlGUY83C7DeeivdvdDqupB47YHLDDbn2eYtv6jSR6V4Wd4R5JnVNW9uvu3FpgtSQ7q7ufNXjH+//G8qvrpBWWadVaSd2b1c7zb7d4oq7pndz9yvPzGqnp6krdX1UKXTxht6+4XJ0lV/fzM8/ziqvqZBeba6WkZCoundfe5SVJV/9bd91hsrFVtnxnN//tVtdAlCkb7VNU+3X1tklt291lJ0t0fmykiF+nz3f3UJE+tqh/MsNTN+6rq/Ayjg15xU974HjkdLEmqaq8MC97OLgx9Vnd/bXGpBlX1yiSv7u5/WGXba7v7cat82W5RVQdkGG3z6VW2PbC7/88CYs1muPn4R8XK6/dPcuedvwSXRVUdk+SBS3CSsktjgbGtu/9t0VmS64cH3yNDgXbJooeM7jSecH5s0TnWsvMVgu7+5Dhy5cFJ/r27/3mhwUZV9Z1JviPDQuQfWXSelarqo939bTd22+5SVV/L2ifz9+/uW+7mSNerFdM1xxP5h2aYMvnWJVhPbnYq7A2mXa/MvghVdWKW+GS+ZqbnrvJcL8Pjd36Sw8b15N7T3fef2XbuEkx1nn38dp7M/3iGaSU3+cn8BvLt6udjoUsBVNUHuvt7Zj4/q7u/bzzP/3B3f/uiso153pLkbUn+ZOe5yjg67olJHtLdD15gvFTVh5L8t+7+11W2XdzdBy4g1myG8zMs4XHdzHVPzFBw7Nvdd19gtuu/96rq2d39jJltC/+9MuY4IMnvZ3ix9HcyTP9f9IsGSZKquiTDFLDKsAbpPXssFmpcwmDB+X4xw4sZJyf5oQxl/V8l+b+SfGt3P36B8VZdi7eq9s5wrvDo7n7S6l+5NZZpBMONMv4yec+ic6xmnEay1raFFUDj7V+yi20LLYDGDF9XAI3XX57hldSl0t2nZ3j1eamNI6uWogBKknGa3wcWnWOlZS6AkqH8mbn8hQxrei2N7j4vyXmLzrELn6iqX8/qJ/PL8I5w5yf52bVO5heQZ9bNq2qvnSfy3f2cqro0w0Lg+y42WpIbTm//0xXb9t6dQVbT3S+oqtdneIV058n8Mr0K9y1V9WsZTuZvW1W182Q+y7F0wMuSnFHDGg9vqqoX5r9O5t+/yGArdfe7k7x7/APkIRnWl1toCZTkP6rqRzKMFu6q+rHufmMNU+kW/eLpNVX1A939D+PokM8lw3l+VS3DCOZHZ3gn03eO/190kssyjBJ+1CKDjZ6ZtX9Gf3E35ljL32b4OX3bziu6+4/HEXMvXliqwd9U1b7dffWKAujgJB9dYK7rjX+3/cT4s/HWDMsBLIv/nWGJgiT5kyT7J/nsOML6/YsKtVN3v7iGpVj+R5J7Zeg9DknyxiTPXmC0nb7ub45xMMub8l8jc28ye+xIIAC4Marq9hlO5o/NsMBn8l8n8yd398opvLtVVR2X5Nzu/rqTz51/tO3+VNff/vOTvKW737bi+qOSvLgX/IYMVfWsJM/vcc27mesPzvDcHreYZF9vPJn/rQzTTO606DxJdr4z56yX9fCmAnfK8Lg+YRG5ZlXVEbnhyfzFGU7mXzUO91+YqtrR3Y9ZZIZdqarvSfL8DNPqfzXD4/hTGUbRP6W7F7ZUQVV9d4bpxIdkeBHhp8fpGndM8tjuftGisu1UVd+eYWHj98z+jqmZhXEXacx31wxvoLIn5Tu6uxe93t0e89hlKGzv2d0fWsZ8y/74ybfitpVAAExdVT2pu1+96BxrWeZ8y5wtWc58Nbzj5c6T+aXLN0u++ci3ecuQrap+KcNUl/MzrM32y939N+O2r5vOsbvtAfl+McN6rUuXb5mzjRk8t3Pw+K1z+0ogAKZu5ToZy2aZ8y1ztkS+eck3H/k2bxmyjdNJHtDDG6kclGEa9mu6+4WLXk9Jvm/cbPLJd1Pn22PXBAKAG6OqPrjWpgzv6rhQy5xvmbMl8s1LvvnIt3nLnG20185pGt190Tgt8S+q6u7JUrzrqnybt8zZEvnmJd8uKIEAmIptGd6GeOXaP5XhbZMXbZnzLXO2RL55yTcf+TZvmbMlyWVVde/ufn+SjK/aPyzJq5Is/N2jIt88ljlbIt+85NsFJRAAU3Fahrekff/KDVX1jt2e5ustc75lzpbINy/55iPf5i1ztiR5QpIbLDzew0LkT6iqly8m0g3It3nLnC2Rb17y7YI1gQAAAAAmYK9FBwAAAADgpqcEAgAAAJgAJRAAAADABCiBAAAAACbg/weGJBIrdvtbOwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 1440x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "roc_metrics = []\n",
    "\n",
    "for i in range(n_labels):\n",
    "    roc = metrics.roc_auc_score(test[i].values, outputs[:, i])\n",
    "    roc_metrics.append(roc)\n",
    "    \n",
    "s = pd.Series(roc_metrics, index=range(n_labels))\n",
    "s.plot(kind=\"bar\", figsize=(20, 5), title=\"roc auc score per class on test data\", grid=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "026466a4",
   "metadata": {},
   "source": [
    "Test some sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "afdfc814",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = transformers.SqueezeBertTokenizer.from_pretrained(\"squeezebert/squeezebert-uncased\", do_lower_case=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "431fda9c",
   "metadata": {},
   "outputs": [],
   "source": [
    "def score_sentence(text, topn=5):\n",
    "    max_len = 35\n",
    "    with torch.no_grad():\n",
    "        inputs = tokenizer.encode_plus(\n",
    "            text,\n",
    "            None,\n",
    "            add_special_tokens=True,\n",
    "            max_length=max_len,\n",
    "            padding=\"max_length\",\n",
    "            truncation=True\n",
    "        )\n",
    "        ids = inputs[\"input_ids\"]\n",
    "        ids = torch.LongTensor(ids).cpu().unsqueeze(0)\n",
    "        \n",
    "        attention_mask = inputs[\"attention_mask\"]\n",
    "        attention_mask = torch.LongTensor(attention_mask).cpu().unsqueeze(0)\n",
    "        \n",
    "        output = model.forward(ids, attention_mask)[0]\n",
    "        output = torch.sigmoid(output)\n",
    "        \n",
    "        probas, indices = torch.sort(output)\n",
    "        \n",
    "    probas = probas.cpu().numpy()[0][::-1]\n",
    "    indices = indices.cpu().numpy()[0][::-1]\n",
    "    \n",
    "    for i, p in zip(indices[:topn], probas[:topn]):\n",
    "        print(mapping[i], p)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "40c4abbb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sadness 0.79904836\n",
      "neutral 0.16807877\n",
      "disappointment 0.13121246\n",
      "caring 0.04404119\n",
      "remorse 0.03200784\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[W NNPACK.cpp:51] Could not initialize NNPACK! Reason: Unsupported hardware.\n"
     ]
    }
   ],
   "source": [
    "score_sentence(\"I miss my friend\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}