notebooks/06_protein_analysis.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "t_gnhLD0HvhU",
    "toc-hr-collapsed": true,
    "toc-nb-collapsed": true
   },
   "source": [
    "## Summary\n",
    "\n",
    "In this notebook, we test the accuracy of a network trained to reconstruct protein sequences on the *test dataset*, comprised of sequences of Gene3D domains excluded from the training dataset. We also evaluate the accuracy achieved by the network at predicting the ΔΔG of mutations, a task that this tangential to the objective that the network was trained to optimize.\n",
    "\n",
    "----"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "eoBMUoW2Hvhp",
    "toc-hr-collapsed": true,
    "toc-nb-collapsed": true
   },
   "source": [
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "TbKxMUZWHvhq"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/strokach/env/lib/python3.7/site-packages/Bio/KDTree/__init__.py:27: BiopythonDeprecationWarning: Bio.KDTree has been deprecated, and we intend to remove it in a future release of Biopython. Please use Bio.PDB.kdtrees instead, which is functionally very similar.\n",
      "  BiopythonDeprecationWarning,\n",
      "/home/strokach/env/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:526: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/home/strokach/env/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:527: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/home/strokach/env/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:528: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/home/strokach/env/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:529: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/home/strokach/env/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:530: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/home/strokach/env/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:535: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n"
     ]
    }
   ],
   "source": [
    "import tempfile\n",
    "import time\n",
    "import warnings\n",
    "from collections import deque\n",
    "from contextlib import contextmanager\n",
    "from pathlib import Path\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import tqdm\n",
    "\n",
    "import matplotlib as mpl\n",
    "import matplotlib.cm as cm\n",
    "import matplotlib.pyplot as plt\n",
    "import proteinsolver\n",
    "import proteinsolver.datasets\n",
    "import pyarrow as pa\n",
    "import pyarrow.parquet as pq\n",
    "import torch\n",
    "from kmbio import PDB\n",
    "from matplotlib.ticker import FormatStrFormatter\n",
    "from scipy import stats\n",
    "from tensorflow.python.summary.summary_iterator import summary_iterator\n",
    "from torch_geometric.data import DataLoader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "try:\n",
    "    inline_rc\n",
    "except NameError:\n",
    "    inline_rc = mpl.rcParams.copy()\n",
    "    \n",
    "mpl.rcParams.update({\"font.size\": 12})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "UNIQUE_ID = \"191f05de\"  # No attention\n",
    "# UNIQUE_ID = \"0007604c\"  # 5-layer graph-conv with attention, batch_size=1\n",
    "# UNIQUE_ID = \"91fc9ab9\"  # 4-layer graph-conv with attention, batch_size=4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "BEST_STATE_FILES = {\n",
    "    #\n",
    "    \"191f05de\": \"protein_train/191f05de/e53-s1952148-d93703104.state\"\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PosixPath('/tmp/strokach/ml_data')"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "DATA_ROOT = Path(tempfile.gettempdir())\n",
    "DATA_ROOT = Path(tempfile.gettempdir()).joinpath(\"ml_data\")\n",
    "DATA_ROOT.mkdir(exist_ok=True)\n",
    "DATA_ROOT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PosixPath('protein_analysis')"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "NOTEBOOK_NAME = \"protein_analysis\"\n",
    "NOTEBOOK_PATH = Path(NOTEBOOK_NAME)\n",
    "NOTEBOOK_PATH.mkdir(exist_ok=True)\n",
    "NOTEBOOK_PATH"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PosixPath('/home/kimlab1/database_data/datapkg_output_dir')"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "DATAPKG_DATA_DIR = Path(f\"~/datapkg_data_dir\").expanduser().resolve()\n",
    "DATAPKG_DATA_DIR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='cpu')"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/home/kimlab1/database_data/datapkg_output_dir'"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "proteinsolver.settings.data_url = DATAPKG_DATA_DIR.as_posix()\n",
    "proteinsolver.settings.data_url"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "dsEY3dtLHvhy",
    "toc-hr-collapsed": true,
    "toc-nb-collapsed": true
   },
   "source": [
    "## Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "datasets = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "DATAPKG_DATA_DIR.joinpath(\n",
    "        \"deep-protein-gen\",\n",
    "        \"processed\",\n",
    "        \"validation_data\",\n",
    "        \"part-00000-4f535e50-cdf4-4275-b6b3-a3038f24a1a9-c000.snappy.parquet\",\n",
    "    ).is_file()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "datasets[\"protein_valid\"] = proteinsolver.datasets.ProteinInMemoryDataset(\n",
    "    root=DATA_ROOT / \"protein_valid\",\n",
    "    subset=\"valid\",\n",
    "    data_url=DATAPKG_DATA_DIR.joinpath(\n",
    "        \"deep-protein-gen\",\n",
    "        \"processed\",\n",
    "        \"validation_data\",\n",
    "        \"part-00000-4f535e50-cdf4-4275-b6b3-a3038f24a1a9-c000.snappy.parquet\",\n",
    "    ).as_posix(),\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "datasets[\"protein_test\"] = proteinsolver.datasets.ProteinInMemoryDataset(\n",
    "    root=DATA_ROOT / \"protein_test\",\n",
    "    subset=\"test\",\n",
    "    data_url=DATAPKG_DATA_DIR.joinpath(\n",
    "        \"deep-protein-gen\",\n",
    "        \"processed\",\n",
    "        \"validation_data\",\n",
    "        \"part-00000-4f535e50-cdf4-4275-b6b3-a3038f24a1a9-c000.snappy.parquet\",\n",
    "    ).as_posix(),\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Protherm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "file = DATAPKG_DATA_DIR.joinpath(\"deep-protein-gen/datasets-test/protherm/protherm.parquet\").as_posix()\n",
    "\n",
    "extra_columns = [\"qseq\", \"residue_idx_1_corrected\", \"residue_idx_2_corrected\", \"distances\"]\n",
    "extra_column_renames = {\n",
    "    \"qseq\": \"sequence\",\n",
    "    \"residue_idx_1_corrected\": \"row_index\",\n",
    "    \"residue_idx_2_corrected\": \"col_index\",\n",
    "}\n",
    "protherm_dataset = proteinsolver.datasets.ProteinInMemoryDataset(\n",
    "    root=DATA_ROOT / \"protherm\",\n",
    "    subset=\"\",\n",
    "    data_url=file,\n",
    "    extra_columns=extra_columns,\n",
    "    extra_column_renames=extra_column_renames,\n",
    ")\n",
    "protherm_df = pq.read_table(file).to_pandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Data(Index=[1], edge_attr=[7356, 2], edge_index=[2, 7356], x=[162])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "protherm_dataset[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>filename_wt</th>\n",
       "      <th>chain_id</th>\n",
       "      <th>mutation</th>\n",
       "      <th>cartesian_ddg_beta_nov15_cart_1</th>\n",
       "      <th>ddg_exp</th>\n",
       "      <th>cartesian_ddg_beta_nov16_cart_1</th>\n",
       "      <th>cartesian_ddg_score12_cart_1</th>\n",
       "      <th>cartesian_ddg_talaris2013_cart_1</th>\n",
       "      <th>cartesian_ddg_talaris2014_cart_1</th>\n",
       "      <th>ddg_monomer_soft_rep_design_1</th>\n",
       "      <th>local_filename_wt</th>\n",
       "      <th>structure_id</th>\n",
       "      <th>model_id</th>\n",
       "      <th>qseq</th>\n",
       "      <th>residue_idx_1_corrected</th>\n",
       "      <th>residue_idx_2_corrected</th>\n",
       "      <th>distances</th>\n",
       "      <th>mutation_matches_sequence</th>\n",
       "      <th>qseq_mutation</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>/home/kimlab2/database_data/biological-data-wa...</td>\n",
       "      <td>A</td>\n",
       "      <td>G44S</td>\n",
       "      <td>-1.808667</td>\n",
       "      <td>-0.53</td>\n",
       "      <td>-0.701</td>\n",
       "      <td>0.088</td>\n",
       "      <td>-0.289667</td>\n",
       "      <td>-0.633667</td>\n",
       "      <td>-2.384</td>\n",
       "      <td>/home/kimlab1/database_data/datapkg/adjacency-...</td>\n",
       "      <td>107l</td>\n",
       "      <td>0</td>\n",
       "      <td>MNIFEMLRIDEGLRLKIYKDTEGYYTIGIGHLLTKSPSLNAAKGEL...</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "      <td>[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 66, 70, 91...</td>\n",
       "      <td>[1.3463744749991646, 4.7287436401778065, 6.389...</td>\n",
       "      <td>True</td>\n",
       "      <td>MNIFEMLRIDEGLRLKIYKDTEGYYTIGIGHLLTKSPSLNAAKSEL...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>/home/kimlab2/database_data/biological-data-wa...</td>\n",
       "      <td>A</td>\n",
       "      <td>A120M</td>\n",
       "      <td>2.617667</td>\n",
       "      <td>-0.20</td>\n",
       "      <td>0.354</td>\n",
       "      <td>0.560</td>\n",
       "      <td>-0.069000</td>\n",
       "      <td>-0.188000</td>\n",
       "      <td>2.472</td>\n",
       "      <td>/home/kimlab1/database_data/datapkg/adjacency-...</td>\n",
       "      <td>160l</td>\n",
       "      <td>0</td>\n",
       "      <td>MNIFEMLRIDEGLRLKIYKDTEGYYTIGIGHLLTKSPSLNAAKSEL...</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "      <td>[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 66, 70, 91, 92...</td>\n",
       "      <td>[1.345760255620924, 4.727237880524551, 6.42311...</td>\n",
       "      <td>True</td>\n",
       "      <td>MNIFEMLRIDEGLRLKIYKDTEGYYTIGIGHLLTKSPSLNAAKSEL...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                         filename_wt chain_id mutation  \\\n",
       "0  /home/kimlab2/database_data/biological-data-wa...        A     G44S   \n",
       "1  /home/kimlab2/database_data/biological-data-wa...        A    A120M   \n",
       "\n",
       "   cartesian_ddg_beta_nov15_cart_1  ddg_exp  cartesian_ddg_beta_nov16_cart_1  \\\n",
       "0                        -1.808667    -0.53                           -0.701   \n",
       "1                         2.617667    -0.20                            0.354   \n",
       "\n",
       "   cartesian_ddg_score12_cart_1  cartesian_ddg_talaris2013_cart_1  \\\n",
       "0                         0.088                         -0.289667   \n",
       "1                         0.560                         -0.069000   \n",
       "\n",
       "   cartesian_ddg_talaris2014_cart_1  ddg_monomer_soft_rep_design_1  \\\n",
       "0                         -0.633667                         -2.384   \n",
       "1                         -0.188000                          2.472   \n",
       "\n",
       "                                   local_filename_wt structure_id  model_id  \\\n",
       "0  /home/kimlab1/database_data/datapkg/adjacency-...         107l         0   \n",
       "1  /home/kimlab1/database_data/datapkg/adjacency-...         160l         0   \n",
       "\n",
       "                                                qseq  \\\n",
       "0  MNIFEMLRIDEGLRLKIYKDTEGYYTIGIGHLLTKSPSLNAAKGEL...   \n",
       "1  MNIFEMLRIDEGLRLKIYKDTEGYYTIGIGHLLTKSPSLNAAKSEL...   \n",
       "\n",
       "                             residue_idx_1_corrected  \\\n",
       "0  [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...   \n",
       "1  [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...   \n",
       "\n",
       "                             residue_idx_2_corrected  \\\n",
       "0  [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 66, 70, 91...   \n",
       "1  [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 66, 70, 91, 92...   \n",
       "\n",
       "                                           distances  \\\n",
       "0  [1.3463744749991646, 4.7287436401778065, 6.389...   \n",
       "1  [1.345760255620924, 4.727237880524551, 6.42311...   \n",
       "\n",
       "   mutation_matches_sequence  \\\n",
       "0                       True   \n",
       "1                       True   \n",
       "\n",
       "                                       qseq_mutation  \n",
       "0  MNIFEMLRIDEGLRLKIYKDTEGYYTIGIGHLLTKSPSLNAAKSEL...  \n",
       "1  MNIFEMLRIDEGLRLKIYKDTEGYYTIGIGHLLTKSPSLNAAKSEL...  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "protherm_df.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "%run protein_train/{UNIQUE_ID}/model.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 1\n",
    "num_features = 20\n",
    "adj_input_size = 2\n",
    "hidden_size = 128\n",
    "frac_present = 0.5\n",
    "frac_present_valid = frac_present\n",
    "info_size= 1024"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'protein_train/191f05de/e53-s1952148-d93703104.state'"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "state_file = BEST_STATE_FILES[UNIQUE_ID]\n",
    "state_file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "net = Net(\n",
    "    x_input_size=num_features + 1, adj_input_size=adj_input_size, hidden_size=hidden_size, output_size=num_features\n",
    ")\n",
    "net.load_state_dict(torch.load(state_file, map_location=device))\n",
    "net.eval()\n",
    "net = net.to(device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training plots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0007604c.xxx  44a43c56.xxx  8a5f95bb.xxx\n",
      "191f05de.xxx  4c3e6ed8.xxx  91fc9ab9.xxx\n"
     ]
    }
   ],
   "source": [
    "!ls protein_train/runs/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/strokach/env/lib/python3.7/site-packages/tensorflow/python/summary/summary_iterator.py:68: tf_record_iterator (from tensorflow.python.lib.io.tf_record) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use eager execution and: \n",
      "`tf.data.TFRecordDataset(path)`\n"
     ]
    }
   ],
   "source": [
    "dfs = []\n",
    "\n",
    "for tb_file in Path(f\"protein_train/runs/{UNIQUE_ID}.xxx/\").glob(\"*\"):\n",
    "    data = []\n",
    "    for e in summary_iterator(tb_file.as_posix()):\n",
    "        for v in e.summary.value:\n",
    "            row = (e.step, v.tag, v.simple_value)\n",
    "            data.append(row)\n",
    "            if v.tag == \"accuracy_mv\":\n",
    "                row = (e.step, \"wall_time\", e.wall_time)\n",
    "                data.append(row)\n",
    "    df = (\n",
    "        pd.DataFrame(data, columns=[\"datapoint\", \"tag\", \"simple_value\"])\n",
    "        .pivot(index=\"datapoint\", columns=\"tag\", values=\"simple_value\")\n",
    "        .reset_index()\n",
    "    )\n",
    "    df[\"datapoint\"] = df[\"datapoint\"].astype(np.int)\n",
    "    dfs.append(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "28793\n"
     ]
    }
   ],
   "source": [
    "df = pd.concat(dfs, sort=False)\n",
    "print(len(df))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "27784\n"
     ]
    }
   ],
   "source": [
    "df = df.sort_values(\"wall_time\", ascending=False).drop_duplicates(subset=[\"datapoint\"], keep=\"first\").sort_values(\"wall_time\", ascending=True)\n",
    "print(len(df))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert len(df) == len(df[\"datapoint\"].drop_duplicates())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAVAAAAEMCAYAAACSpO5VAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nOy9eZwcV3mv/5yq6nV6do32ZWTJ8m68CIOxsVkcAgEDwYQtBIeEkMAvAbLcXJKQjZuFm5Ab4IaQACEJhCXkAgk4LCGsNngVYNmSjax9HWn2mV6rq+r8/nhPdfWMZqQeaWY0ks7z+UjTXV3L6eqqb73nvMtRWmssFovFMnecc90Ai8ViOV+xAmqxWCxniBVQi8ViOUOsgFosFssZYgXUYrFYzhDvXDdgvli2bJnu7+8/182wWCwXGNu2bRvSWvfN9NkFI6D9/f088sgj57oZFovlAkMpdWC2z2wX3mKxWM4QK6AWi8VyhlgBtVgsljPECqjFYrGcIVZALRaL5QyxAmqxWCxniBVQi8ViOUOsgFosFssZYgXUYrGcG6IIRvefvNwvnX7b6gSUhqFekff1qtlnCEd/OG9NPB0XTCaSxbLkiYuXK7U4xwt8cFMzHzOKAA2Om7RttnZFETgz2Frx9/FLoEPIdk79fHIAiidg1bXyvjgo67WvNO8HYPQAjB2ClVdDuh0O3m/aAqBgxVWQ65a2jR+Bkb2w+jo4+iPQESgHll8BJ56QdfouE3EFWbcyBsu2QLpNPq9XIZWdw0k8NepCqUi/detWbVM5LUuaffeKOFxyW2vrFwehMgp9W1rY93eheyPUJmHFlbJs73eNKGpYcTXke8RqS+fh8COyfONtcoyBx2Hjs+HEk+B40HMJTB6T/QzvkX0Wj0N5GDbcAge+h6ic0Y8ogjU3wLFHp7YrCmDltXBiR7Ke4yYiGQayD8czyxyojoOblvXTeXMMJYIZi3lQEzGOjHBXJ+Rh4WXEKk3lZJsolOM4xlbUGtbeKILaIkqpbVrrrTN9Zi1Qi2UhaLbaolCEASD05SYOfRh4TIQq3yMC1rFa1knlYfKoWFw6gq51RvAQkQvrcPABWP9MY7EBkQ8j+0SwysPQdzmENUCJiB7fkbTFccw2gYh6vHzfvbJvEPHUkbwOarK9UuCXRTzrFfCyENVBeRBU4NiPRBCjwHx3F2rjsm29It8ZwEvLcXQo29YmINMulmymXaxW15Nz5q2VbZWS91GYnMeoLu1J5aAyIsuzXfLaTYv4Oinwi5ApyLHCOhz5gZzHecAKqGXpE9aTruhciIyF4qVP/iweO0vlpm0TiXC4026NMJAbMdcl1l7bcigNyg3ftR6GdsnfwkpjnTX17IwBBUpu/H33GuurLiKqHFleGTHd4tjaMmJ28AFQrryPBc8vwYH7oV5M2ufWxZpsXyVd2ngs0fFkn1EA9bIsy3TKMUJfjh/4IlK1SVmvrQ/8SRGi0AcdyHaVMXkf1qBWlG2iQM5nrlv2VZuUdVI5KJvv5BfNcWrSHsdJxC8+Zq0o3zGsQmiGEyYHIKjK5+mC+U4KXFfaX6/B+KFkDFRHZp2SWLKOJ9v6JWmHDiHb3tLl0wpWQC2Lz+RxKCyXC3r62Fp1fOpYWmkITuyUrubBB2S8y/HEsjj6A7HCwiARvGbL78D35O/Gpi5zWJeb+dhj8l4p6ZI2b6N1YiWtvl6sl+OPS9suuV1u1tKgiEg1gmPbZT+DP4ahp4ylZQRAm7HGKBQRLA8nbYnqgCPCUB4WyzMWiup4YpEFVfl+Tgq8nHSBfSNetWIiTsqVbdyMdGWr48ZSTEO+V85lQ0jNWGAsaFEgx0EnD6xaUQS0XpK/QVU+q1fkr+OKFRrWxQINqqYLbcSsXhHBKh5PzmngA5XkHNQCaVt5GDrXwsRReQgVlkt7J4+efG1URozAarG0B5+Uc9e+SqxwNKy4Rr5TaQjKg+aBqaW9HWtbvlRPhx0DtZwdgz+WgftmJ8S+e2HdTfJa66mD9vu+a4TTTbqIsdj0bJTxtnU3iZNg5TUikmEIm54De7+djJVlClAZF0ukQTwmp2SdKJA2OW5yDK0TUatXEudC8+e1yUS4mi3Uellu1NqkeV+RdgS+CIxfTPbrZUQ03bSIi0asOL8s5yMKRQx1IM4TfxJwZN1YUKLQiJwvbXZcyHRIm0qDIib1ijmu+e5uRkQrtmJ7NsHwbvmOOpRjOG7SFYbkd9OR2U8k+wnN9wpr8nm6XX6j4gkR+Xo5EceVV5vr4UnIL5Pz4mWTcxoFcq0svzwZj5wJvyTbOe7U5VEoQt18LcVDI4cegsowXPZT0mYnNbtDrHhCxmp7Ns7ehmmcagzUCqhldsYOSheqsEJugN5NUz/fd68Z7/KmCmgYyA2gzE284RZ5PfSUWBg6EgsnqouwxBaQ44lFlMrJjeBljGXky3hWbVIELPTlJqsXAVesR+UCpkvqZpLuq5um0eUD8Cuyz3jczM1ArlNEWiHHr4xJd9QvQq7HaJMWaybbLiIY+qb7a47vmPG1oGa64ea+clPmpjbfsW05lE7IdrFguSn5TGvxOg89Jd3Y5VfBxBHZxstIe4sn5FzGx9ahOIhUkyUf1uX7p3LGEjVe6WxH8jvF4qPU1IdKFMi5HdgOPZth37dFJPsul/Ny2Ytk20MPioivuBrueTu88p+gbZkcp16GJ78MP/gnef/cd8FDfy/W/NY3JQ+QyQHZ//gR2HwHfOOPpl5fV7wUei+F44/B7v9OHrg3/xrc/3/ndi2/7EPwH2+R17/6CCy7tOVNrYBaZubEk2I9NluO8Y2/8Tbx4kYB4ohARHHt0xOHRhTKmFumXYTDTcn2sWc1lRcLKdsh76NILKvY4lNKRDTVZm5oJTe+UiKaHWuk+6VcEdDqhAiRUvLXLxrBzEoXV0fG4eCKJefljFCOiqUZBYnzIpUzXcxIxLJeFrGtTUpbMu3yueMmDo7YwaIjEZ3CcgnFocmCJRKvc0x5WBwbsTWrpg1ZgAiJUmK5gYzpdayWcxpberHXuDoubfSyMqwwcVQcPsu2iEgBvP7zMLgLdnxeHFTPeIuc55F94rQa2C7fddXT5Lt+76/lNxg/JJ8BrNkKR87gflp7Exx+aO7bLSb9z4afv6fl1a2AXsxMd8DETohll4p3tHeTdAUHHjfWXkWsMjctQlgZMhs6kM6ZG7pqrMWqWCeZjmRbz1iPgS/duNAXgYq7vaFvrLQoscoyhSQu0TdODiLI9UrXNhbX2GpTDolQm+4mOhG22JrCeG51aI4Vh7yM0ejyZjrFUwyyrH2ldFWHd0HnOhGVWGy9LBRWSVsyhannuTYh7SgNiUNpzQ3yN9cN97xD1rnmZ6R9lVHY8w0Rxbv+ET79qtZ/zxt/AbZ9rPX1LSfzy/fBqmtaXt2GMV0MRJF08U48kYRoxBZlpl3EJ9uZeF3Hj4iojR0CDopABL7xgE6YbrsvXV5tLDc/B3ljRboZs9xPHCB+WUQqME6EsglbCWpJCItyZN/5XslCCWumi2kEr2O1iE4UyngaQO9m6T4GVTNWiAj5ssvEwTG6P7Ek0wX5O53SUNLFZP3pz2emQx48vZfKGGKmUzzTcxG76Tz2b1Pf+6W57+9iEM8rXipj3+OHpy6/4Q1iWR96QN6vf5b0Qm79dfltKqNiQa+6DnZ8AW64W4YI7v0rWf/qV8LG209++J0FLVmgSqk+oKK1LiqlXOANQAj8i9YNT8A55aK3QOOg6XpF4gZ7LoH99wEqGZCPQzyCmohpcUC6yaEvguo4sn5Qk7GzOAwFkgDlhgNCyfqRCW9x0yI2J3ZM7cKCOIZ6N5nus7E8t/0TXPlSufBj5wycnBETBUn8X7Mlvfc7YkXHsZPlYRlaWH4l/PjLEnzupKRrO344cUh9839JlzrXlaQR3nC3tP/xzyXxhOcr2S5jYc8R5TQ59aax7hkyDJDrTq6l4nH43C/Kg+bVn5Qx0YMPwJ7/ls/vNl3k2HKPf9PR/fKbhb44lWoTImozDW3MF41eC+fGiaSUehD4Fa31D5VS7wHuBOrAt7TWv95ySxaQi0ZAG+OUZhww1yXWVVgTwSgeF6eD4ybxdW4G/AkRsCiSp3a6TcQ0XZD91YyjAQcwN5LjiaWYbpMxMvTUC700lGx/fKdkeAzvhsMPi2W77iZY90xxGhRPwAMfhCPbZv5e/bfB/u9OXXb7O+E775nHk7eE6bsc+q6AnV+Q9898qzhp8n0yNLLtH0/eprDSjMEiY5Z3/JG8Dutiia28VoTriS/BlhfJNRE/hHZ9VWJQcz2w+fnQ3S9WfjqfZAEFVZOFdJW8P75D2jndQz4TWk8duz0TolDGbr3s7OvE4VjFEzImDVMF8/jjco1290ubJo5I6NsiC+go0KO11kqpw8CzgCKwQ2u9quWWLCAXpICGdXEw1CsijCBCGccUOp50o6NILtagJt2YfI9YmH5JbgovLa+jAHCkSzyyRy7ueGxy5bQxoRNPwFf+B2z+Cdj9dbFob7hbrMxMAR7/fOJlvRh5/h9B9waxwHQozpyuDXL+o1CswExBxlFj4rCgGL8kv+Ncc7OrEyKQ/bfO/HmzgJwJtUl5GMftGngscaq1r5R9Tw7I9+neKKLUtUGus/i4A9tJahU1hat1rZcHeiqb5NKDXLtxvGy9ZI5xTM5P13rpoZzYKet2rpNjF48nY9yuidttduY1IhxMxEFUl/2tu0ms0BaZDwEdAtYAW4DPaK2vUko5wLjWev7C+s+C81JA4x/c9Uxlmn1yIzYTe8ED410uDYrwaZ0EOPtF2T6sIxeOYy74UrJ9YYU8idOmqzxxVG6I3k3SlaqOwQ8/MXN1nIuJS18AT/0XvPYz4hD7zGvheX8gD5DIZPp0989tn1EoQwPZrtNnVEWBiRHNnPzZbMVIwrrJunFF2OJhlY7V0oPQkViP+R4zZFOHwSdElDrXStgUiMh090v4WpwFBXKNRSZywS8l4U9uOoleiB19USBWc3VM1tehiKPy5LsHleRB3sjWioeDTAZY/D2jQMatyyPyHTIdoo+1oomkMNZyVDdhZ2UacayxkKbb5P6Ires44H4OqZzz4UT6CvBZoBf4jFl2JXCk5VZYphKFkoqnkHG6oV3iVY6DoIOa/OCVMbE047E5v0gjra82ISE8oQ9EMv7neHITlQal69Wcgx348M8vmdqOvivkZlpsnvFWePBvT15+229LW9Pt8K3/Jcte9QmxiCYOy1jbY/8msaWXvwS+/wGxVK59DVz7KunSpfNJmFNlzNzASqwlSOIo65WTKwgBPOttyevXfXbqZ/H6sYW14ipjbXbA6F5xNuW6TFyoOcb4YXlfGpQ2DD9FEvQPU6IFQPbbtQ7GDiSfS8AsiTVn1muON9WmKIeXkWtp4ghMHJPvm8rKuYgzoTId0rahXdJFdjMibiN7pFsf+kmxjkxHUzKCEU7XvHdSIsx1I4xBTY7RvsLkyoeJ4zLOZiqegLZe47g02UvpgomS6EjSbKvjcuw4u6mwXPaV6YDykIiml5Nz4KZEHOPz3ngIOfJXR/K9UvmpvYCzpFULNAPcjYx7fkJrHSilngOs1Fp/5pQbLxLnhQW67155wnetk4wcvwxoeUoGNXnSBhXj0W6yLh0vyVvO95qc4UBu3oHHZB8rr4XPvkGO89K/ge2flrGvxeSWd0i7trxQLnjXM+I/JoLf1jfzduVh2P5ZuO5nTczoDMTZLGeSEz/TvqApA6fJcRWFIhxxBECuV36T7n6xzuOUQCclD7KwlnQTHZMc4LhJoLtyEodcaZhG2JWbSoTdTcvnVRNO5aZMqFhKXrspeVh6GZOrb4Q2v0yWRya/20kZcUtJN7w8KmIe1uWhEtaSONfOtdIeHcr1l24XK7A8kghVUEXE3QwzxKFrjmt0WycOSa1NOqcv36MrHrowGVDKTRISquNT43C1FjGMaxBEZggifjDFv0/XOpOI0dT7alwPTZlYcaUmx5PvOX448RGsuKopGqM15i0O1HTbV2itj82pBYvAkhfQfd+Vi8txjQUZSpB4WBdRDH1zcTTFLDZbKN390i0HEU0dwRNfhK5+CMpidR5++Ozb2XcFXPNK8VaDOIFu/+3kqd08vqa1WMaOZ57s8yRu8Y0TOytqRRneQEnKYCOzJ0wC68FYhUoscRDratmWpJRa7xYR4dG9gCP7GthOI6MHaFh58TBJ3N2Mrfx0Qb6n44nIZQrGsjJiGNaTrm2c157OiRNuckAejG29cqi4x+GZ+NrACIrjmMwoY5l6OclechzwTCaW4xiP+zji3POaimSoxCJ0XBG9dJsISFCV85nvSYqBeFlTcKUpAkNHTQ8V85uvvDqp6hQT+nDJc+WBUx5KxK1nswhVXEikY420Zf99yb7jaA4diie+PJQMBww+KX97+hPrM9+THDd2pipHgvfjPHs3JcMspUEZwsi0SwKBX5Kx1Nke0KdgPsZAu4C/BV4J1LXWbUqplwI3aa3fNecWLQBLVkBjr/fhh0w2i0+jJmJlFFBJvnNQEUtyckCstenFbu97n4SJbH0TPPLR+Wvja/9VbsRcz/wVm43FrHMdjB9M8rvjh0EsiKmciE3HWlPWbXtSASi2KpRrgvLrppqQEbnmdEgi2U65xiGBWDNhbLWarCdlHAuOserctIhLvSKi6KbESdO+UtaJiw7XJuU3ynYmDpA4CUCTWDhhTfapkd8zUxAxdD35rJHrnZJldZO4MOW3NudQI7G4bsakiRrBjq1kN5P0TLSW0K3u/qQASyzCsSd7wy1w4D4RpDhNVpmU20tul33s+66MizuenIMoStJspxd+CQMpp7fx2bJeUDFFjEdh/c0zP1C1TmKFq+Pmd9YyxtxMFMl+4ofNTDT/ZgvIfAjoZ4BR4N3ATq11t4kN/b7WuvWk0gVkSQrose1ykUSBdGP9koip1uaG8pHiDiaectllMp7Xtkysg8+/6ezbcMcfw5obk1zrZottNrSWdqTyYg26GbEOCstl/Cr22I8fgUKffLc4nAZoxJ7GmUYpM5Cfyhpnl9/UJTRRBc2ZSW3LknTF2oQZ68okJcmUK1Z3ptOMETumG9hlxK4oFmxQMbn15rjxel4u8fJm2iXsqzZuxuF8qExAx6rku8Q5/XH1pEapOyUhMX5JfuvV10lNzLY++Xdip1jDJ3aKE694Ikl5VQo23CrVn9Y+PWljVE8KkUwclQfN2ptM70JLiu3QbqlS5GWT86ccWLY5iYsFEUPlivi2r5BKVoO75LeKb/vll8v3KJjhlfJIUgHeAsyPgA4Cq7XWdaXUiNa6xywf11rPMAq/+Cw5Ad13b5IT7hflaRrUJPhba7HGCiuScaDPvObsj3n5S+CmX+akeM3TEQUmRKTJsRHXkIy7Wqlc0r3zy0BkbuB4WKIpv90zFlqcyllYCaXjYuFWRuS8pHLJ51FkSq8Zj27X+qSoyMRREe64FFw6L69rkyK0pSHjdCqIGGoNhEk1oXi8TblJEkBcDSiOg3WbiploLSLmpiXcJR66qJsCIgOPyQNkerbTTFNFTJ8KY9+9cvx0XkRzLiI1W23T+FrKdJz8WXz84zul5kFz/OaB++W7rb2x9TZcpMyHF34cWAY0xj6VUuub31uaGNknT/3KqLyPYzSzHXKTlkemWpdPe92ZHecVH5VuVuzYiLN5aLoxdWQKWeQTh1Nz2bfYu+tmxbMaF9YtLDf1IFPJWF69YtJCR+TGzHQai9rE4eW6ZPtYJOL9d61NurbZbtmflzHdwnTS3HyPrO94kDZe3ly3dHfbeiXm1UnL9nHx3ly3WMm5TindVps04TgbpMxZtsPkyhucuIusZBuUdG3r5SRj5vgOEZzmUKJ0G9A2tbZoMzMNfUzv8sYhNXGpv7nguDMHsCs1u1MkPn48xUcz625iynViOSNaFdCPAp9TSv0e4Cilbgb+DPi7BWvZ+YjWMkgehWLdlE0B27j794Vfnnm7Rz81+z7jghbXvgauf31iycYTc8HUmpXVcRHLwSdojBXGY4peznj5c0m3T4diGcaCmgpFSHJdiVc3CmU/XsaI2TKx2ty06W6aMbbCclPHMkpu9t5N0q0cO5RM/qXTpgv7LDOGVxexi7dZ9wz5/Nh2+Xz100zoignlcjy595df2VRwV0HnmpPPn5uWgPN4QjGYWeya86PjSdDmmzkEby84rWQTWU5Lq114BbwdeDOwATgI/D3wft2iG18p1QP8A/ACYAj4Ha31ScqhlHoN8MfASqCGxKD+mtZ6Yvq6zSyJLvxek4o4eUxEoTYuzpHhPfDNPz799k97rRSFHTso4hBUE8/7TJQGk4m/2lclAdvKSawpvywCVh6WcbnigFiBsagqlcTxxY4Jx8xHEztodCj6OmWaCy3pg4cflnXWPT0R8rgg8ZS2DicOgXgooJnR/eJoat4u8OVcLNucLKubquluOhGBffcm44jNaM2MVe8tljmwJMrZKaU+jZhEvwhcB/wn8Cyt9Y5p661DCpcMKaUKiFAPa63fNn2fzSwJAX3qvyWEY/wIoCRl8lOvnH39Z7wVHv2khHhc+XKxyE5FPFaZNU6V4oBYm8qVKkWOJwKU6xGxC0zpuGyXbO96Zp6b3FQLpPcSM+xgpsboWife3LjE3MB26c6vuvbkwPORfdLN75y/aRIslqXEWY+BKqWeN9tnWutvtrB9G3AXcLXWugjcp5T6IvBzwDun7e/QtM1DYDNLncqYeHDLw2K5HXoQvvbOk9d77rskVfC23xLxu/ynZt5fFMq44fBuCfEIahLe5GZEpGsTJi1tJaBgtCwWnJuS8UcvB0yIVdjdnxQPiUuErd2ahOJkO8RiHj80NcUtttxWXz/7955DUQaL5UKj1THQf5j2vg9IA4eBS05e/SS2AKHWelfTskeB22daWSl1K2KhdgBl4KdnWe/NyLAC69e3UONxIRnYLnP0BD58733SvW7mzg+IkCknCX2J62QqRwQurgYOJAHdBbFoo2BqALZyREDTbYDphueXiaNl1XUiinEYS7PIzSZ4PRutGFosc6QlAdVaT7mzTE3QdwGTLR6ngHjymxkHZhzg01rfB3QqpdYAvwTsn2W9DwMfBunCt9iWhcEvJ6Eu08XzZz8385hfnOIWF0ZI5UxFdxMLWC9JqFNYl/1mO03MuOlaO654hfd/Lylhh5tkW8zmMbZYLPPCGVWk11qHSqk/RSzQ/9PCJkXEmmymg9MIsNb6iFLqq0gBkyXkwpzGvnvFojz0EGyfVhogLixbGZUuMo6pCtMklDoSq7KtV8YrGxOslWXdKEpCeADc5lQVoP8WEzw+f5W2LRbL6TmbKT1+gqmlYU7FLsBTSl2qtTa1s3gasOMU28R4wKbTrnWu0KagwsTh2cUzTmuMPcdxXcV4QjUwsZMFU5R4j4xlloclCyX04egPk9CfmUJQrHhaLItOq06kQyTJXwB5IAu8tZXttdYlpdTngXcrpd6EeOFfhhRmnn6snwXuBQ4hk9f8KfCNVo5zTth/nzh0vvHuqctf/Nfyd+CxpGq74yWlwrKdSWZJGIrDZsVV4kVffd3UfaWy4tHPtNv4PYtlCdGqBfr6ae9LwK7TxWZO463Ax4ATwDDwFq31DpPRtBO4Umt9EKkz+r+BbiT//svA78zhOItPaTApYptug9d8mkZFHC9rAtUN6UKSCeN44vwpD4un+1QxnzMFiVsslnNKq06k75ztgbTWI8DLZ1h+EHEyxe9/D/i9sz3eohFU4d6mYeDX/qv8jcukta+U7nVcOahr7cmVZywWy3nJrAKqlPoEU7vtM6K1fsO8tuh8IgzEeRNXIooD1kHGOzMdYnE6nnTLB3clVdEtFst5z6ly3HYDe8y/ccR6dBHPu4OMYZ7B/KkXEAfvhy/+avL+VR8Xy3PgMRHPtl4R0o3Plu752httmTCL5QJiVgtUa91I3lZKfQ14sdb63qZltwK/v7DNW6LUK5IDHhdLjpk8lsztkmmXKuG5rtn3Y7FYzmtarbLwTOCBacseBG6e3+acJxx+RDzp2z6WLLvrY+IMypkphdMF8ZjbHHGL5YKlVQH9IfBnSqkcgPn7p8CPFqphS5Z4xsBaEXb+R7LcywLKFPPokq76xmdPncfFYrFcULQqoD8P3AKMK6WOI2OitwIXnwMpnritOV3zVZ+Qsmu5LonndLw5zTttsVjOT1oNY9oPPMvEbK4Cjpnwo4uPeEKseH5tkEnI3JTko6OkiLDFYrngOVUYk4qLJZvpjEE88Iebl2mtW03nvDCIAqmO9O0/T5bVzIyWXsbMRng2GbIWi+V84VR3+jhJARAzO9oU4gl1Lo7cwjCQlE2/BBNNU0Hd+QEz73ZeJoyz4mmxXDSc6m6/qum1jf4+eL+ZMmMM7ntvstwvSh3OdPvUQHqLxXLBc6o40ENNrw80f2a88KHW2l/Ati09ysNwz9uT95kOCZTPmyk0ZpqszGKxXLC05IVXSr1XKXWTef1iYAQYU0rduZCNW1KEgVifzbzk/RLz6WWg33rdLZaLjVbDmH4WeNy8/gOkOtNLkamNL3yiSOY7ap5++O57oHhMBHTDrTZF02K5CGnV45HXWpeVUr3AJVrrzwEopTYsXNOWEBOHJYD+iJn1M10wVeNTZhpgO22uxXIx0qqA7jKFjjcDXwdQSi0DKgvVsCWFX55asu7OD8j0woWVtsCxxXIR06qAvhV4P1AHfsEs+0ngvxaiUUuKoAaHt8HAo8my4nH5m+uGdc84N+2yWCznnFYzkR5m2vQbWutPAp9ciEYtKUYPwIF7k/fX/IzEfIZ18bx76XPXNovFck5pefBOKfUTSql/UEp9ybzfqpR63sI1bYkwcRSe+JK8vvouuOFumXo414OZm8NisVyktBrG9GvAh4CngHiy8QrwJwvUrqXB2EFJ04xrft5wt4QyOZ503ztWn9v2WSyWc0qrFug7gDu01u8hmcr4SeCyBWnVUmH0gIx/xihH5nZP56FrvZ3byGK5yGnVidSOTDMMSU58CrhwM5GiCMIafP/9ybKB7RLClMrDss3nrm0Wi2VJ0KoF+l3gndOWvQ341vw2ZymhoTyavP2Zj0vcZ6FPLCd7KI8AACAASURBVFGLxXLR06oF+mvAl5RSvwS0K6V+DEwAF24q54Hvw31/nbyfOAztqyDVBv23nrt2WSyWJUOrAnoceLr5twHpzj90QdcCjQI4sUNeP/u3xHGU7YbezTZt02KxAC0IqFLKBYpAl9b6IeChBW/VUmByIHm9/pkSzuR60Lnm3LXJYrEsKU47mKe1DoFdQO/CN2cJ8e+/krwe2i1FQ5Zfce7aY7FYlhytekM+CdyjlLpbKfV8pdTz4n+tHkgp1aOU+oJSqqSUOqCUet0s692tlNqmlJpQSh1WSv2FUmpxy7wP/jh5/YI/gVwnZDqhbdmiNsNisSxtWhWmt5i/fzRtuQZaDYb8IBL2tAK4DvhPpdSjWusd09bLI3GnDwJ9wBeB3wLe0+Jxzp7J48lr5UileRu2ZLFYptFqLvxZTemhlGoD7gKu1loXgfuUUl8Efo5p4VFa6w81vT2ilPok8NyzOf6cmTja9EZBpgBd6xa1CRaLZemzWAGNW5ApQHY1LXuUqfMuzcZtwHQrFQCl1JuVUo8opR4ZHBycaZW5UxmFbzfVifYy2Jx3i8UyE4sloAVkls9mxpEMp1lRSr0R2Aq8d6bPtdYf1lpv1Vpv7evrm5eGohwYM1NAXfMqcR6l8vOzb4vFckGxWM6ZIskUyTEdwORsGyilXo6Me96htR5awLZN5VBTlNa6m6Tq0tobF+3wFovl/GGxLNBdgKeUurRp2dOYvWv+QuAjwJ1a68cWoX0Ju5pqRMczblosFssMtFrOzpnpX6sH0VqXgM8D71ZKtSmlbgFeBnxihmM9DwmbussE7i8eUQQPf1he92yS7KPeTYvaBIvFcv7QqggGyHQeU/4ppWpKqX1Kqb9SShVOs4+3AjngBPBp4C1a6x1KqfVKqaJSar1Z7/eBTuDLZnlRKfWVuX6xMyIKktc/9VcyHuraivMWi2Vm5lJMJB6TPASsB34b+E/gx8AfAu8D3jTbDrTWI2Yf05cfRJxM8fvFDVlqJqgmrwd3QvtqUHbSOIvFMjOtCuhvADdorWNP+i6l1CPANq31JqXUY8C22Tc/Txh8sumNglyX5L9bLBbLDLTahe9AMoSaySNdbYABpHt+fvOxn5S/m54vhZMdK54Wi2V2WlWIjwNfV0q9H+nCrwXeDvyz+fwFSFf+/CauzrfiKuhcLdN2WCwWyyy0KqD/A5lQ7jXAauAYktv+EfP5t4Bvz3fjFpWoqbRpx1pwM1ZALRbLKWk1Fz4C/s78m+nz6kzLzytKTamgqSx4WVs42WKxnJKWB/mUUi9AqihNCVfSWv/BfDfqnBA7kHLdMv5prU+LxXIaWg2k/xvgX4AbgXVN/9YuXNMWmdgC7Vwr+e+F5ee2PRbLeUwYaUq14JTrbDswSi0IAdBa4wcRZf/U20xU69y/Z/iM2vSDA6PsPlE8o21no1UL9LXAdVrrQ6dd83zlyXvkb9tycFO2+265qHlw7zBb+3twnan3gR9EpL3Z7a7hYo1dxydpy3hMVgMcpVhWSOM4ilo94srVHUSRzIxe8QOOjlXpLaTZcWQCrTUaeNamXh7YOwLA9eu7yKZcyn7Ao4fGAIUfhNy/ZxilQGtoz3pMVuskVdM0ruMQRpotKwrsOi6iGUYR1boI9ublp8v7aY1WBXQYGJuXIy5Vdn9T/m64xU5bbFkU6mGEqxSOM/eH9f17hrlxQzfbDoxw/fpusimXKNJM1gI6c6nGevuGSjgKxsp1rl3b2RCmp/d3M1T0OTJW4cYN3VP2C5p6GPHQvhHWdOXIphw68yl+cGAMrTVKKaSWurT7mZf0MFEN2Hl0graMSxCK9VnxQ1Ke4vhEFa0hQnP/njoQi1nEsbEKR0bLKKWo+CFBFPHA3hFKfoDnKB7aN0zKdYm0ZrhYo689w1i5TnsOHAW1ekgYRYyUfFxHzmUUaTKeCwp2HZ9krFwnk3IJw4ggkvYvtoD+FfBJpdSfIzN0NtBa752XlpxraiZHoH3luW2H5bwkERYIwgilFI6isWz6uuOVOk8cmwAUN2+aOt3YweEyvYU0Gc/BUYoTkyIcJT9gx5EJnt7fTRBGbDswih9E/PBgbNtowkisr0tXFNg1MIk2n0SR5oG9I/hBiB9EPLx/hHoYEUTw/d1DjXaW/QAN1OoR7VnYO1REKXCNwFXqIbm0Sz3UZDwHreF7u6VYWjWI8MOQUi0kijQT1To9bWn8IKJSD1EoOnOeOU5IqDX1yCOKpN2jZd+cHzg+UaWQ9QgjTU9bGq1huORTyHgMlWqUfLFua0FEPh1QqgWEWhNLu2u+T09bmpGSTzblNqzPrnzygDlbWhXQuEr8S6Yt18CFleuYKdgCIhcQWmtqQUQ2NffLdN9QiYzncHyiSnvWY1NfgUjDQ/tG2NrfTcp1GlbfzqNxkp5Ca5GtZvHsa0/T39uG5zrcv2cYDQQmdO7+PcONbQCq9ZCDI05jH0EUsW9IuqS1IOLh/aPUgohQa8bKPt1tacJQM1L20Rr62jM8eWyC45M1OrIe1XqE1iJ4k7WAYjVgeUeWyWqdSGvaMymyaZcgjBiv1MmnPSp+yETVRxkr03MUdWM1UhLrDyDS0Jb2cBxFEEb4YdToooeRZlj71IIQbQzWSj0Qa9R014vVANdR+GEEGlZ15TgyVqa3LcN4pU5ofj+Qh8DOYxPyYEIxWvZZVsgwUamzridPNuUSRpq0q9h2cIwb1nXxiQcOcOvmZYxXfI6MVrlhQzdruuYv56fVMKYLu0/bdPHipqBj9blri2VWakEoXTNEGLXmlN3f8XKdoVKNExNVnnlJb0PQokjz4D7pyt60sYdqPcRRCo0m0vDY4XFWd2UbXcswiqj4IQPjEq0XRJoH9w7jOopIS1tKtYBMysVzEkstn/FwgJIf4gchg5M1Ig21IKJYrZNNuVT8kLQXgumOZlMux8ar5NMujlIiNJpGt3y45LOskGasXCeMpKtdrsn2FV8sLD+IyKQcxst1Jiv1xjlSyLDB5uXt7D4hpXi1Wd+pOI1zWqpVCSLdsOKauXxlBygRxxOTVXrbMnxn1wkynsOyQobHj0zw7Ev7GK/4rO7K8eTAJO//xlMA3Lihmzc+q59f/fQPG/v7y1dey76hEn//3b2EkT7peP/whq0cHa/w+/8xY+XLlvjazqTT/PEHDvCZX3oG63vbznh/zajmJ9/5zNatW/UjjzxyZhuPHoD3Xwv9z4Zb3wGb75jfxllOi9YaP4waAhl7Y/NpecZX6yE/PDiK6zhcvqqdJ45OEBmzZmt/N4/sHwXghg1daC3W4+BklZTrUvID8im34XRQCir1iDCKyHouQaRJuSIWruNQ8QM8VxFE0h2vBRGusbAiranUQ1KuSyHjMlLyyaRcqn5Iu+lyVoOQci0k5TlkPIdSLaAt45HxHCIt3cuxinQrSzWxwLSGUGs8RxGEJ9+TzVZtbFXnMx7re/IMTtZ4/IhYwNeu7SLjOXxp+1H2D5fZ1NfG1g3d1IKInUcn+PLjAyft+50vupz2jMeewSIf+97+ef1dlyK/eGs/v/+SVmYTEpRS27TWW2f6bFYLVCn1Va31C83re4EZlVZrfVvLLVmq7Pqa/J0cAMeWr1sstNYMl3yeOj7Jys4cR8cqXL26k4lqnUMjZTRw9ZpOdh6dIIwi6qHGdSK2HxpDKUW5FqCU4pH9oxRrdVzH4WFjWSqlODZeZV13nhMTVQoZj2zKZbIWkEu5DJd8IiOcQSTClU25FDIeR8cqFLIefl26yfUwwmmyxsJIgwopVhVhJIKKphGS4wcRkelKf+OJE9ywoZvhok/Gc0S8/RDPcfjDL+7kpv4euvIp/mvncX7jji38n//eNdOpap0HD055u/tEka/tOD7LysJ7vvLkKT8/H1jdleXoWJLP05VPMVauT1knl3J5wZUruPvm/nk77qm68B9vev3ReTviUsTLyt9LXwida85tW85T7t8z3Ag5aV4GmvW9bRwcLuM6ijXdObKew67jxYbjpRZEHB2rMFmts/3wGJHWjJR80p7Ljw6OEkbidNFAxpOxwNA4KbROHAVtGQ8F5NIe9TCkVAs4NlGl4oeNcTStNeOIJdqdT+O5ipTrUA8jBidrPDkwQV9bhsGJGh/45m5+76euYLRUZ/PyAr/+2R8R9zLXdOU4MlZp6dz8x6Myy6ujYHov9aH9I43XZy2eS4QtKwp05dM8ZB5mf/Pa63lyYJJvPnmCnccmeN+rr6NYC/je7iG+8vgAN23s4a3P2US1HpFLuXiuYmiyxtv/9UdT9vupNz2jMQxTrAW0peVam81RBzSGWDxXRiEHJ2vz+l1tFx7ga78L938QnvsuuOR2mQvJAmC8uskFGoRR42KERCRrQdL9jkNcirU6xWognlgTsuM6Ckcp6pGmXAtwXUXVjN2NV+p05lKNMUTHUWRTDjqCahA2vKvKjMFpDVtWtLPz2DjZlDulG1zxQx7cN8K3fiwJEm+4eQMfv/8AN6zv4gcHL7yIvDuvXcWXth+bsizjOY0HB8Bbbt/EjRu6qYcRhYyHH0b85mcfZaySWGrveP6lPOMSiQoYKtYaY7EpRxFqZo0BHS7W6C1kpiybHpkQme1jx1E25cwofvEDsyufmmL5+0HE7sFJ0FDIpihW62YcWjRMm/+2rGznqeMSgeAocYNJr0F89Ldf2sfmFaecz3IKZ9SFn7aD1wI/0lo/oZS6DPgwEAJv1Vqf//Z/xdxQbcth2ZZz25YFJor0rI4XrTW7jk8Cig29ecJIm+BlcdaIaGm5KM2FPVmt4yhF2Q9IeyEKqIURDop6GDFS9ploCnJ2HYXniMNGR1Ax3V6A1Z05Do2UeXj/CMs7stywrpuhUo0/b+pivvGWfvYPlRrCOJ2bN/XOmKny8ftlptWlIp63XbqMF169in/83j4yKZefvm41/7XzOO+4Q66/ih/iOJBynJbjRF/3jA2N1zNFApyEgg+9/uQJE3ceGwctlvyJiSoo8BxxMoVag4a1PXnGynWKtToKyKZcjpt1FclYc+x995rGeV1z/YRR1BC95pnD4wft8YnKSZ/35OVhXA8jClmRr/ghn3IdIq3Zc6JIW8ajkPHwHOnhaC1Waz7tMliszUlAT0VLFqhSag/wLK31caXUl5DSdUXgNq318+alJWfJWVmgf/dsGNgOr/wnuPJl4Fx4QQex5Xj/niGuWdtFIeMxMF5l31CR+OqMn+RBGDUcG9V6SLEW0FNIM1L08cOIfNojm0q6RGnXoRJEhMbJEj/5S7WAGzf0SHfcdfjS9iNs6ivwxLFJ7t87xG1b+vjSo8dmafHSYGVHll+5fRMfuXcvb3/+pazryVOsBRwYLlGrR1y/vgul5GFxYLhEZKzi6QRhxERVHGMS1yjnaaxcpyPrsXeoxPqePHuHirRnUvS0pQkjTSHr8eSxCdlJHOQI9LRlGClJd7RZqJpxlGqEFLVnUyZbh4bINbz8yHDGaEniMHNp14QWaVyVWI3tuVQjZlMsW013WxpXORSyLiNFv6F1fiiipdF05dIEph2O6T20ZTxGS37DWZf1HCr1kIznUMh4jWOMVupkTbyp5ypWdGSJIk3ZD0h5EhyvlCJlhmJqgVyDnqtIu9IjqtZDUq4i1JqUI1bv9NjbU3HWFijQZ8QzC9wKvBKZF2nxphteSJQDKEhlLjjxjLtRD+8fYWVnDj+IeOzweEMsi9U6adP1Hi7WyKVdyn4o3mBzEdfqYWO8Mb6wFYngfv4Hh3nkwBj/+65r+J+fa30S1YUWz5sv6eW6dV1cu7bTCOAWvvr4MX7qmlWNYYjmbuapeOeLLqfXiJrnKPoKGbIplxOTNcp+yIqODJuXtxNEks+9f6hEJuVSq4cN4bukT7JfJqp1Do+WcZXCcx0GJirk0y4Hhkt0ZFNU6iFHxiq4juLwWJl8xiXriahNVgPzGwWkUy6eAxnPbcRl1oKItrRHNQjJeq4EtwchGk1nPi1ihIiR5yjGKwGuklCrXMolnXLpyadIm+5/tR7RmfMYcmp0t6VxENENzdhiHCUB0N2WbgTXB1FErR6hHEVb2jPJBTS65EqpRqJAPYpIuzK2nfRuNAVknZQZNvAcxQ3ru4m0ZvthSeskJffuFavaGa/UG46kK1d10GkC5u/fM9yI482l5zdsfS4W6E8C1yCTwb1AKZUHjmitu0+99eJwVhbo/71RRPRlfwPrnjG/DVsEIiNqORMMvWewxLJCGs91eOzwGK7jmFhHGs6WahAyVq4bT3TiHY5vdsdRhGHEkbEq33zyBFtWtPP1J8Sb++JrVvHsLct45xzEci688ZZ+tu0fZbsJzfkfL7iMHccm6M6nePzIOENFn/fcdQ3etIddFGnGq3VGSj4bevN4jjiHUkYsdx4dn9LFXNGR5fhEVeIiByebMqlP7obGY22u4xjrXIv33nVwlcSG1sMIz3HwHIXnKsJQk045jR3GueGOo2hLu+TTLtmUy0S1TnsmxVCxRm9bmni4TiGCWMh4eK4CJPypK5eSLKJQfvcOEyOqtabihxSyKcJIehFxbGXcdXbMGLJ8N+lGN9YxwzsZz+Xyle08OTDZeMBorVnRmaU7n+bHA8YibsqimqjWOThc5uo1nWit+cHBUfxAA5pc2mNlR5YgiljZkW2cgyjSDJVqTFQCNi8vUK2HpN1kyEJ6SCVis/uGDd1N4+yLx6ks0FYF9OeB9yPjnq/WWn9dKXUn8Jta6+fMY1vPmLMS0PdugfZVcMcfw6bnzGu75pOJap2U40x5ih4Zq3BgqGS6zRCEciPHg/2SN+xxYqJGNYioB6HENEaa9myKjqxHPdD82md+OMtR55ffuGML9+4eZE1Xnls297K8PUvKFasjakrFg+RFZGJEm60dkK7Z3kEpFBGLQl97hmWFDEfHqoxXfDxXxu5i8ct4kj2kHAhN11HGxjy68ym02a/rOJT9gPasJx7fjEfVDxsOsSDUjVRNz1Em3EmTdh1Snoin66rG2GE91FTrYaNGTZKvrozYOdRDCW+abhFfsaqdrnyaehjx8L4RrljVQSHr8Yjx4K/tztPdlubQSJmxcp1r1nbiKtW4TsJI8+OBSS5f2Y5jxD9OQoiTCprF9pmX9JzUhumWemBCu84kj/9846wF1OwkD6C1Lpv3ywFHa31yZO454KwE9M9Ww8pr4Tm/I174JYbWmmItYPvhcdNVE5mJQzQGJpL4t3oYNfKUI60p+yFVP2S07NORS/MvDxxgshpI7OI88fPP6ueWzcv4wy8+zqtuXMea7hzd+RRKqSmiF5mMHT+QcSvXUbRnPfYPlfBch1KtflJ2UfxKKQlkjxe6SjXCmlxHkXYd/NCkGppqPCDhRpEplRY7Gfwgoj3rSUZNKMLgOJB2JQ50pFTDdZSJPlDJmLDZ37Hx6eFLU9M3V3VmOTZe5fKV7XTmUgSRZtuBEXMO5Ds1j8PFjr3pFphlaTAfXvg+oKK1LiqlXOANiDX6L/PXzHNEFIJfglR+yVZhemDvcCNXuFqXcJ56qKnVQ9KepOzFHBuvMFENmajU+eqOs3+2renK8aZbN5JLu6wzWS/f2z3Ev207zF03rOWVN65l51EZU/3LVz4NrWHX8QkmKnXSniNpgk7izMilpEqOo8TiOTZeIeM6RNoMQUSwsiPDZDVoiEncnS3WQvJm/K8zl6IearKeQyblJmmIfoDnOI20z+62NBU/JJfSDStVZ+CatZ1EkW4U9FjekWGTGaN8eP8I3fmUjGmGEdUgoi3tNjKl+pedPg2weZ20o7h50zKARkGLdFMoWCyYZ5Kvbzm3tOpEugf4FeCHwJ8CdyJOpOuBX1+Ypi0SVVMEIj0/YQ1nQrEWkPWcRnczLjkGSYrjiYkabRlxWsQOnDDUfH/v8KwhPXPh1VvX8bLrVje6aYdGyqzoyJL2nEZ1oTDSdOVTPP+KFbzihrVEkXRLsymXYyaoXCloy0gxio5sSkQn5VCuSfc1tggVimItQCkRDlcpamFE2lWkXJeUK2OJsRUImkLGE6s14xknkIhyZy7N+t48jx0epyMrlm82JZbpdeu6AOnGKpIyr/H3jIWtmaf39zRee65DwYjdfIy/WZG8sGhVQLcAcVrA64FnIWFMO7hQBDRzbqYx9oOIRw+NGe+kdMsnqxJb55iA8DDSjJVr/Pbnfswrrl/D5394ZM7HUQqeubGXn7+lnyDUjS52XHYs7Tk8cWxC6iaaEmJxwYnmQPrYijsyVkYBKdehr1080kEocaK5tOSId+fTDcHqzonoX7uui+Giz8B4lc6ctGHjsjZCrTk4XG50ax8/Ms6qzuxJwdn1UJweMwnRMy/pkXqPQKkWNryw07+DxTJftKoYIZBWSm0BxrXWB5VSDtPmRzov8UvyN922aEH0JTOeCTSKU8Q1E1OOYqhYa4R71MOIP/1yEkjeini+5xXX0N2Wpj3jNWIyIxMAXawFjFfqDExUGrGAsZc1djpoR7KIcmmXXEo8xbGnPggjImQS00hLF1oCmhUZD0CzqjOH6yg2LS/QnknhODBakqDrjmyKjmyKjTN0g5vLjF29pnPG75ZyHWYz4uJ4QIDO/NIcjrFcWLQqoF8BPgv0Ap8xy64EWjaFlFI9wD8gc8gPAb+jtf7UDOvdDbwNuBSYAD4F/K7W+tSTpZwpFdNdTufl3wITi2exWidlKvVMVOuU/RAdae55bIDdJ4oUTzOfTMx7XnENG6aV5ir7AQdHyg3RjMcco0jGAbOeS0c2RWcuxWQtoD3jUQ8jcmkPzxFPvmvGHuNQIaUU/cvy7B8qN+UZy3pb+3saoUIx08cJV3a6QPbMTprFskRpVUDfBNyNjHt+wixbBvzRHI71QcAHViCze/6nUupRrfX0Qn954B3Ag0Af8EXgt4D3zOFYrXPCWHepPKTmr9DqdKJI8+jhMSp+SLkeMlzy0SaH/PPbjjRiHk/Hb9yxheGSz3Mu6yPtOjw5MMHRsQpjZb/RvfZch7a0R2fOY6xSpzufblQVAgmh0Vo82JmUQ9p1G2Eqm5a3sedEiZ62NBuXtTUC6uOwqFWdco7CSHNktMK6nlxLgegWy4VIqwWVa0j+e/Oyb7d6EKVUG3AXcLXWugjcp5T6IvBzwDun7fdDTW+PKKU+CTy31WPNmcBUZ0ktzGiEH0SNeL1aEDFa9qUSeKTZdmCMex47fTbOZSva+cM7rySINE8dn+Rp65azb6hIaGI5y3XJIilkPUp+QF97lowrTinPdRqhRFPiIVMOV6/uZMxU9fbDpGr78vaplmJ6hvFD11Gs7114i91iWcqcqh7oh7XWbzavP8Hs9UDf0MJxtgCh1rq5XtejQCtBl7chzqqFwTP1P3Mzj7mdCUfHKmQ8h4MjZUq1gPGKxF3Wg5B7HjvG93bPPi3rS65dxe1b+hoilvYcKn7IEwMTOErRkUuxd6hIb1uaWhA1nCyuKTzRHUkG0pquHCs7s2w7MDqlzNz0YiLLO+Q4Wcd6hy2WuXIqC3Rf0+vdZ3mcAjC9jzoOnDJ2SCn1RmArMoQw0+dvBt4MsH79+jNrmW/mifYyp15vDuwdLOIoRTWIGC/7jJR9qn7IX/7XzPUeX3vTel5yzSocR+EHEUPFGrsHJ8mlPFMd3SGXculpSzeKQrRnpbCD5zps6M2TT7tSCs5Yo0mYztSiCTZI22KZP2YVUK31nze9/uOzPE4R6Ji2rAOYnG0DpdTLkXHPO7TWMxYt0Vp/GDO0sHXr1jMrbDpu/GCFFWe0eTOxkybUMtFYpRbwG/+2fdb1n3/5cn7yqpWs68lLCTHEaeOaAgxxNZ6M59DdlibtxjnKE6RchxUdWdZ2Tx2DnJ7uaLFYFo6W7zalVD9wLdNCl2bypM/ALsBTSl2qtX7KLHsas3TNlVIvBD4CvFhrvTAVK2LqZVAuZLvOelcP7B1GKSjXQnafmOSvvv7USevcsqmXN9+2qeGUKfsBO4+NS5C4UnS1pRvz4jiOTI0rAeVOw5qcKfjbYrEsPq2mcv4O8AeI4DUnAmskzOiUaK1LSqnPA+9WSr0J8cK/DAnIn36s5wGfBH5aa/1QK+07K7SWcdB6+ax2E0aaehQxMF7l3x45zCMHRqd8ft26Lu6+uZ+ufIrBYo1x4zVPuQ4rO7KkXXHspFyXjcvaTBUaAM3yjiyXtJA+aLFYFpdWLdDfBG7UWu88i2O9FfgYcAIYRsri7VBKrQd2AldqrQ8Cvw90Al9u6preq7V+0Vkce3ZKQ+BmIPTPeBf37xkmjCKGiv5J9TDf9rzN3LChG61lbHTSFMzoLaRJuUlJM4D+ZflGmNDKThszabEsdVoV0GFg/9kcSGs9Arx8huUHaRoW0FovXMjSTARlcNPQufaMNp+s1gmjiO2Hx/mLr/14ymfPvayPrf09PHV8Es9V5DMuywoZsp7b6J6nPZcbNyyJkqoWi2WOtCqg7wA+rJR6H2JBNjACeP4SheKBHz8MPRvnvPnjRyb43u4hPvSdvVOWf/qXnsnOo+PsGyrSkUvRlU+RS7mNscxTzU1ksVjOD1oV0DSSgvm6acs1cH4HENYrMq2xO7f54MfLdXYem2Cy6k8Rz7957fV059M8cWycTMplXXcOlJQvu3RFO8tM3KYVT4vl/KdVAf1b4HeRPPjWJsM+X6iOQ6Ydujecfl1DFGl2Hhtnx5EJ/uTLTzSW/88XXo4fRPz4+ARd+TTd+TRd+TQ9bWnW9disHYvlQqNVAfWAf9Raz18Z86VCUIXCcpg4Cu0rT7t6GGke2DvMWNmfIp4feM11DE7WqNRDVnXlyKVc1nbn2Lz83NUZtVgsC0urNb/eC7xTXYhVI+pl6cJ397e0+oN7h9k3VOL/+1Qyh9A/v/EmBidr0mXvydGVS5PxXCueFssFTqsW6NuAlcDvKqWmJHJrrc8wh3KJENREQL3Thw0NFWvUgoh3/fvjjWW/cMtGdp+Y/bBh+wAAIABJREFUpKctTU8hzeUrOxr55RaL5cKmVQF9/YK24lwRBhL/mcrBaYppVOshO46O88Z/erix7OO/8HT2Dpboa89QyHjc1N877/NOWyyWpUur5ey+s9ANOSfMoZDIDw6McvfHEvF86dNWsWewRF8hQ09bZkpFd4vFcnFwcVeeiGuBumlwUrOutu3AKO//RpLXfsP6bq5f383KjiydudSM82hbLJYLn4tbQCMzHbCbAmdmf1o9jDg4XOLBfVIU+YpV7fzCLf2U/ICufNqKp8VyEXNxz7wV57+r2Z8jj+wf5dc/+2jj/StvWMt4pU5fIcPVazqseFosFzEXtwUaNlmgM/DU8Uk+94NDjfd/cde1APQU0nTkUrRnZ+/2WyyWC59Wy9n1IBO7XcfJ9UBvW4B2LQ4NAZ35NAwVff7fNim4/KvP3UylHrK+J0825bK1v2exWmmxWJYorVqgnwIyyNTGZ1c4cykRd+FncCAdGinz2o880Hjf05ZidWeWvvYMV66aXlzfYrFcjLQqoM8C+szsnBcOp+jC/+f2o43Xr966lu58mkI2xdVr5m/yOYvFcn7TqoBuB9YCexawLYtP7IV3pp6GiWqd93xVanv+5k9sYZWxPG/aaLvtFosloVUB/SbwVaXUPwIDzR9orT82761aLOIuvDe1UtKHvpU8J/Jpl7aMx82bluHaEnQWi6WJVgX02cBh4CemLdfINB3nJ2Egf72kC39opMyHviMCet26TjrzadqznhVPi8VyEq2mci7uNBuLRWyBNhVTHpxMhnl/+vo1rOnKcpUd97RYLDMwl2mNu4E7gTXAEeBLWuvRU2+1xJlBQF/xoe8DsLIjy/L2LJuWF+iw8Z4Wi2UGWspEUkrdjDiQfgWZG/6XgT1m+flLZLrwqZPLz/3y7Rtpz3osb7el6SwWy8y0aoG+D3ir1voz8QKl1KuBDwBPX4iGLQoNC1SqMY2WkqmNu/NpVpophi0Wi2UmWs2F34IE0Tfz/4DN89ucRabhhRcB/e3PbQfgjiuW09eeZfPywmxbWiwWS8sC+hTwmmnLfobzPS409sIbC/TrO48D8Jwty0m5F3edFYvFcnrmMi/8PUqptwEHgH7gUuAlC9SuxSE0HvdUnl3HJxqLewtpnt7ffY4aZbFYzhdaDWP6vlJqE/BiYDXwJeDLWuuRhWzcgtOUyvmrZpK4lz1tNe1Zz5aps1gsp6XlfqrWelRr/S9a678wf+cknkqpHqXUF5RSJaXUAaXU62ZZ726l1Dal1IRS6rBS6i+UOkXBzrOhaQx013GZ3uPWzb2s6bZzuFssltOzmAN9HwR8YAXws8CHlFJXzbBeHhkyWAY8A3g+Ukpv/gl8QKGb9Lm7LcOGHiugFovl9CxKQWWlVBtwF3C11roI3KeU+iLwc8A7m9fVWn+o6e0RpdQngYXJhIp8cDyeGJKu/B1XLCefdnFs2qbFYmmBxbJAtwCh1npX07JHgZks0OncBuxYkFaFdXBcPvrIEADre/J27NNisbRMq5lIb1NKLTuL4xSA8WnLxoH20xz3jcBW4L2zfP5mpdQjSqlHBgcH596q0AfH5eCYWKBXrupgywob+2mxWFqjVQv0DmC/UuoepdSrlVKnn0h9KkVgehn3DmBytg2UUi8H3gO8SGs9NNM6WusPa623aq239vX1zbFJGAvU45EjFQA6cyl6C3P9ahaL5WKlJQHVWr8U2AB8BXHwDCilPqqUanU+pF2Ap5S6tGnZ05ila66UeiHwEeBOrfVjLR5j7oQ+KLfxtpC9uOfYs1gsc2MuYUzDWusPaq1vBm5HcuC/pZTar5T6PaXUrH1frXUJ+DzwbqVUm1LqFuBlwCemr6uUeh7wSeAurfVDc/w+cyMKCI0Hfnl7hmXW+rRYLHNgTk4kpdTzTVX6bwPHgTcgnvTrEev0VLwVyAEngE8Db9Fa71BKrVdKFZVS6816vw90Al82y4tKqdPt+8wIfOomEOEFV61kVZctHmKxWFqn1WmN34vkwo8DHwfepbU+0vT5A8Apa4OawPuXz7D8IE1TJS9q8ebIx9fShV/TmaWQsV14i8XSOq0qRhb4aa31wzN9qLWuK6W2zl+zFokwoBKJgK7qtHU/LRbL3GhVQP+cafPBmwr1Oa31UQCt9ZPz3LaFJ6xRCj0yrqK3kD79+haLxdJEq2Og/45Ma9zMWuAL89ucRSaoUdYpunMuG3pt/KfFYpkbrQroZdPDicz7y+e/SYtIvUIp8ujJe+Qz7unXt1gsliZaFdATSqkp1efN++H5b9LioXVIMUzTk/dIObaAssVimRutqsbHgM8ppV6ilLpSKXUnMqXHRxeuaQtPVPf5/9s79/iqqjPvf59cCJAEEm4J10RulgDaQah0xEvVouCgVkAFBZHWvnTs4HTwffFWDZepjPPqTCutVNFQbdFiFbVURuRDUWhthXag6gC+oFwCBEOUAAmEXJ73j7UTT8JJss9JziXh+X4+55Nz1l5rr99ee+8n6/qsU5pE91QbfTcMI3T8Wo4lQCVuTXp/4ADOeD4RIV1RoabyNGdIJrNTEum2CskwjBDx65G+Bvh379Nu0OpKKjWJ1I7J5sLOMIyQ8V3tEpEOwPk4R8d11kZVN0RAV3SoPsMZkki3CfSGYYSB35VI44CXgRScF6XjOFd0B4CBEVMXYaTaLeU0A2oYRjj4HUT6D+AxVe0GnPD+LgJ+FjFlUUBqqjhDMt3MiYhhGGHg14AOBX7cIGwJ8IPWlRNFVEnUSs6QSN8M2wPJMIzQ8WtAS/nSIfJhEckDMglwAtLmqKlGUKpIIq2jTaI3DCN0/BrQV4GJ3vdngd8Df8H1i7ZNqisASEhIJDnJ+kANwwgdv9OY/jng++Mi8mfcINJbkRIWcWr3hE9IIinRViEZhhE6zRpQEUnEbcmRp6oVAKq6OdLCIk6VM6AJCQkkJloT3jCM0Gm26qWq1UA1zido+yGgCZ+YYAbUMIzQ8dv595/AKhH5EVAIaO0BVf0kEsIiTrXbylgSE60GahhGWPg1oEu9v99sEK5A27Q+Va4GmpSYSKL1gRqGEQZ+B5HanYWpqaogAWdAk8yVnRFFampqKCwspKysLNZSDI/U1FT69etHQoi24Jydv3Pq9ClSgaSkJBLNkYgRRY4ePYqIcP7554f8whqtT01NDQcPHuTo0aP06tUrpLR+18JvIqDfMxBVvSykHOOEsrJyUoEOSYkkmQE1osixY8fIzc014xknJCQkkJWVxb59+yJjQDnbcXI28G3glyHlFkeUl7s98lI6JJkrOyOqVFdXk5ycHGsZRgDJyclUVVWFnM5vH+gvGoaJyCtAAbAw5FzjgNOnnQFNTrbdOI3oI2L/tOOJcO9HS9oQB4ELWpA+ppwqdx34qebKzjCMMPHbBzq7QVBn4CbgT62uKEpUlJcCkN65U4yVGEb7Yc6cOfTt25cf/vCHrRo3XvFb/ZrR4HcZ8Eecn9A2SeXp0wB0Ss+MsRLDiB9yc3NZvnw5V199dVjply1bFpG48YrfPtBvtDQjEemG8+Q0HjgK3K+qK4PEGwE8DlwEdFfViHQWVVW6tfCpqeYL1DD8UFVVRZJ5LquHrz5QEZkpIhc0CLtQRBrWTJvip8AZIAu4DXhKRIYHiVcJrMKN8keM6jPeKHxHM6CGATBjxgz279/PpEmTSEtL47HHHkNEePbZZxkwYABXXnklAFOnTiU7O5uuXbty2WWX8dFHH9WdY9asWTz00EMAbNy4kX79+vH444/Tq1cvevfuTUFBQVhxS0pKmDRpEl26dGHMmDE89NBDjBs3LhrF0iR+/50sAr7aIOwA8AbwQnOJRSQVmAyMUNWTwGYReQPXNXBfYFxV3QXsEpHBPrWFRY1nQJNsMNSIIQt++xH/c+h4RPPI69OFRyYFq6vU54UXXmDTpk11Tfi9e/cyf/583nnnHXbs2FE3b3XChAk899xzdOjQgfnz53Pbbbexbdu2oOcsKiqitLSUgwcP8vbbbzNlyhRuvPFGMjPP7jprKu7dd99NamoqRUVF7N27l2uuuYacnJyWFUwr4HcUvnYjuUBKgQyf6YcC1ar6cUDYdqD5u9oEIvJdEdkqIluLi4tDSqtVpzlNByQ9uyUSDKPdk5+fT2pqKp06uQHX2bNnk56eTkpKCvn5+Wzfvp3S0tKgaZOTk3n44YdJTk5m4sSJpKWlsWvXrpDiVldX88orr7BgwQI6d+5MXl4ed9xxR8SuNxT81kD/B1eDXBUQ9i1gh8/0aTiDG0gpzilz2Kjq08DTAKNHjw66UqoxEipPUUEHEhJtQrMRO/zUDGNN//79675XV1fz4IMP8vLLL1NcXFxXKz169Chdu3Y9K2337t3r9Zt27tyZkydPBs2nsbjFxcVUVVXV0xH4PZb4NaDzgTdF5BZgDzAYuIovt/lojpN8uadSLV2AEz7TtzrfLF8DQKl1ihtGHcEmlAeGrVy5ktdff53169eTm5tLaWkpmZmZqIZUfwmJnj17kpSURGFhIUOHDgXgwIEDEcsvFHw14T0P9MOBLUAq8D6uP/MPPvP5GEgSkSEBYRcCHzUSP2okUB1rCYYRN2RlZfHJJ427+D1x4gQpKSl0796d8vJyHnjggYhrSkxM5KabbiI/P5/y8nJ27tzJ888/H/F8/eB3FD4FKFLVJap6t6ouAYq88GZR1TLcxnQLRSRVRC4BbiDIAJQ4OgIdvN8d/eYTCmWSynFSSTp5uLVPbRhtlvvvv5/FixeTkZHBb37zm7OOz5w5k5ycHPr27UteXh5jx46Niq6lS5dSWlpKdnY2M2bMYNq0aaSktLpZCBnxU/UWkXeB/6OqfwoIGwssUdUrfGXk5oE+h3PKXALcp6orRWQAro81T1X3i0gu8GmD5PtUNbep848ePVq3bt3qRwoAOxeP5XSN8JU7l9Gx/4W+0xlGS9mxYwfDhg2LtYw2zfz58ykqKuIXvzjLTUfYNHZfROQvqjo6WBq/HYAjgT83CHsf1wz3hap+DtwYJHw/AfvLq+peIOKTi4QalGRI7R7prAzDaCE7d+7kzJkzjBw5ki1btvDss8+yfHlDJ3HRx68BLcVNgC8KCMvCLelsk4jWUCMJJKj1gRpGvHPixAmmTZvGoUOH6NWrF/PmzeOGG26ItSzfBvQVYKWIzAU+AQYBT1B/WlObQlBASOh09tQLwzDiizFjxrB79+5YyzgLvxPpH8TN+XwfN/XoT8AuIPJDcBEigWoUQapOx1qKYRhtFL/ORE4Dd4vI94EewFFVVRFps3sSiCo1CYkkJLev7e4Nw4geIRlAdRQDI0Tk33F7xLdJBHU10ETzSG8YRnj4NqAi0lNE7hGRvwLbgK8B90RMWYQRakASoIN5YzIMIzyabMKLSDJwPTALuAbYDbwI5ABTVfWzSAuMFAlag0Z+tpRhGO2Y5mqgR4Cf4waMxqpqnqouwvn1bNO4eaBttgvXMOKGWl+etQwfPpyNGzf6ihsqc+bMYdGiRWGnb22aG0T6GzAOuBj4fyLyqap+EXlZkUdQ1HZGNIxWJ9DBcktYsWIFy5cvZ/PmzXVh8bYNSJNVMG+Z5iBgHXAvbv37b3EORdq0H7jkBCU50WqghmGET7MWRFX3qeoiVR2Cc2F3GKgBtovIY5EWGCkyLvk2WYP/LtYyDCNuWLJkCVOmTKkXds899zB37lwKCgoYNmwY6enpDBw4kJ///OeNnic3N5f169cDcOrUKWbNmkVmZiZ5eXls2bLlrDwHDRpEeno6eXl5rF69GnDr0ufMmcN7771HWloaGRnOd3vgNiAAzzzzDIMHD6Zbt25cf/31HDp0qO6YiLBs2TKGDBlS59W+td3uheQM03Nrt9lbkfQtYGarqokilaO/y4m/bW4+omFEkrX3QdEHkc0jeyRMWNJstGnTprFw4UKOHz9Oly5dqK6uZtWqVaxevZqSkhLWrFnDwIEDeffdd5kwYQJjxoxh1KhRTZ5zwYIF7Nmzhz179lBWVsaECRPqHR80aBCbNm0iOzubl19+mdtvv53du3czbNgwli1bdlYTPpANGzZw//33s27dOoYPH869997LrbfeyrvvvlsXZ82aNWzZsoXjx49z0UUXMWnSJK699lofheaPsNqwqnpaVV9U1QnNxzYMoy2Qk5PDqFGjeO211wBnoDp37szYsWO57rrrGDRoECLC5Zdfzvjx49m0aVOz51y1ahUPPvgg3bp1o3///sydO7fe8alTp9KnTx8SEhK45ZZbGDJkCO+//74vvb/61a+YPXs2o0aNIiUlhUcffZT33nuPvXv31sW57777yMjIYMCAAXzjG99odO+mcDln3bG7qrwNIhkxxkfNMJpMnz6dF198kZkzZ7Jy5UqmT58OwNq1a1mwYAEff/wxNTU1lJeXM3LkyGbPd+jQoXrbbzTcCO7555/niSeeqDN6J0+e5OjRo760Hjp0qF4NOC0tje7du3Pw4EFyc3MByM7+cs+zprYTCRcbRTEMo46pU6eyceNGCgsLWb16NdOnT6eiooLJkydz7733cuTIEY4dO8bEiRN99Sf27t273vYb+/fvr/u+b98+7rrrLpYuXUpJSQnHjh1jxIgRdecNtr1IIH369GHfvn11v8vKyigpKaFv376hXnbYnNsG1KYxGUY9evbsyRVXXMGdd97Jeeedx7Bhwzhz5gwVFRV1exOtXbuWdevW+TrfzTffzKOPPsoXX3xBYWEhTz75ZN2xsrIyRISePXsCUFBQwIcfflh3PCsri8LCQs6cCT7tfPr06RQUFLBt2zYqKip44IEHuPjii+tqn9HgnDWgaSmJ9MvsFGsZhhF3TJ8+nfXr19c139PT0/nJT37CzTffTGZmJitXruT666/3da5HHnmEnJwczjvvPMaPH8+MGTPqjuXl5TFv3jy+/vWvk5WVxQcffMAll1xSd/zKK69k+PDhZGdn06NHj7POfdVVV7Fo0SImT55M79692bNnDy+99FILrz40fG3p0RYIdUsPTh+Hw9vhvEsjJ8owgmBbesQn4Wzpcc7WQA3DMFqKGVDDMIwwOXcNaFLst0Q1DKNtc24bUOv/NGJEexl7aC+Eez/OXQNqGDEiMTGRysrKWMswAqisrCQpKfR1RWZADSPKZGRkcOTIEWpqamItxQBqamo4cuQIXbuGvkPvObuU0zBiRY8ePSgsLGTXrl2xlmJ4pKamBp1r2hxmQA0jyiQkJDBgwIBYyzBagag14UWkm4isFpEyEdknItMbiTdCRN4SkaMiYj3thmHELdHsA/0pbi+lLOA24CkRGR4kXiWwCvh2FLUZhmGETFSa8CKSCkwGRqjqSZxT5jeAGcB9gXFVdRewS0QGR0ObYRhGuESrBjoUqFbVjwPCtgPBaqC+EZHvishWEdlaXFzcIoGGYRihEq1BpDSgtEFYKZDekpOq6tPA0wAiUiwi+5pJ0pAegD/vrfFBW9MLpjkatDW90LY05zR2IFoG9CTQpUFYF+BEa2Wgqj1DTSMiWxvzshKPtDW9YJqjQVvTC21TczCi1YT/GEgSkSEBYRcCrbOBtGEYRgyIigFV1TLgVWChiKSKyCXADcALDeOKoyPQwfvdUUTM84dhGHFHNKcx/SPQCfgMeBH4nqp+JCIDROSkiNTOLM4BTvFl7fQUEKklG09H6LyRoq3pBdMcDdqaXmibms+i3XikNwzDiDbmTMQwDCNMzIAahmGEiRlQwzCMMGnXBtSvAxMv7g9EpEhESkXkuViN/IfgdOUOEfmLiBwXkUIReUxEou5dK5QyDkizQUQ0Fnq9/EN5LgaKyBoROeE5uHksmlo9DX6fCRGRxSJy0HuONzbibyLiiMj3vVWCFSKyopm4cfHuhUO7NqD4dGAiItfg1uRfBeQCA4EF0ZNZD79OVzoD/4xb0XExTvu90RIZgF+9AIjIbcTejaLf56ID8DawAcgG+gG/jKLOWvyW8VRgNnAp0A14jyBTBaPEIWAx8FxTkeLs3QsdVW2XHyAV99ANDQh7AVgSJO5K4EcBv68CiuJZc5C0/wL8Np71Al1xiyrGAgokxXMZA98FNkVbYwv0zgdWBfweDpyOsf7FwIomjsfFuxfupz3XQENxYDLcOxYYL0tEukdQXzBa4nTlMqK/sitUvT8CngKKIi2sCULRPBbYKyJrveb7RhEZGRWVXxKK3peAwSIyVESSgTuA/4qCxpYQL+9eWLRnAxqKA5OGcWu/t8jZSRiE5XRFRO4ERgP/N0K6GsO3XhEZDVwCPBkFXU0RShn3A24FfgL0AX4HvO417aNFKHoPA5twC09O4Zr0P4ioupYTL+9eWLRnAxqKA5OGcWu/t5qzE5+E7HRFRG4ElgATVDXa3m186RWRBOBnwD2qWhUlbY0RShmfAjar6lpVPYP7B9UdGBZZifUIRe8jwBigP9AR15e4QUQ6R1Rhy4iXdy8s2rMBDcWByUfescB4R1S1JIL6ghGS0xURuRZ4Bpikqh9EQV9D/Ortgqsh/1pEioAtXnihiFwaeZn1CKWM/4brq40loei9EPi1qhaqapWqrgAygbzIywybeHn3wiPWnbAR7sB+CbfuPhXXfCwFhgeJdy2uXy4P98BtwMfATYw1XwmUAJfFexkDghvFrv2MwRmmvkCHeNTsxTsfKAeuBhJxzeE90dYcgt5HgM240foE3I4PZUBGDMo4CVcLfhQ36NWRIIOG8fTuhXWdsRYQ4ZvYDXjNe4j2A9O98AG4psOAgLj/AhwBjgMFQEo8awZ+D1R5YbWftfGqt0GaXGI0Ch/Gc3ETsNt7LjYGM1zxotczUj/F9YUeB/4KXBujMs737nHgJz+e371wPuZMxDAMI0zacx+oYRhGRDEDahiGESZmQA3DMMLEDKhhGEaYmAE1DMMIEzOghmEYYWIG9BxERFaIyOIY5S0iUiAiX4jI+xHKo3ajwsTWjNsKumaJyOZI59NSROQBEVkeax3hEIof0gbpckXkTe+5LBKRpX781ZoBjQNEZK+IHBGR1ICw74jIxhjKihTjgG8C/VT1aw0PtoaRUdX9qpqmqtWtGTeaiEi+iMTC9yiq+iNV/Y6fuLHU2Qi+/JAG4We4HYN7A18FLsftJNwkZkDjhyTgnliLCJUwam45wF5VLYtinsY5gqq+qqqv4ZY510NE/kFEtonIMRH5o4hcEHD4PJwv1dOqWoRzA9i8G8lYL4WyjwLsxXnl/hxv3TLwHWCj9z2XBksfccsKv+N9nwX8AfgP4BjwCfD3XvgB3H/WOwLSrgCW4bytnwDeAXICjn/FO/Y5zjXazQ3SPgW8iVtaeHWQ6+kDvOGl3w3c5YV/GzgNVOOW8y1okG5Yg+PHGssTuA74b9zyvwNAfsB56pWXV1aLvDI6AawDeoQa1zs+E9iHe0F/6N27s8rAi9vdK4fjwPveeTcHHP+xp/048BfgUi/8WpwT5UqvHLZ74XcCOzxdnwD/q4lnqvaZeBK3dn4ncFVz98g7lg/8skH53IFbRnoUeLAZnbM8fSeAT4HbYvBO1XPkDIzCvQcX4/wa3OHduxTv+BzgedxOD32BD4FvNZtPrI2HfeoM6NXAq8BiLyxUA1rlvWCJ3sOzH7cuOgUY7z3MaV78Fd7vy7zjP659sXEOKw5450ryHryjeGvAvbSlOKcWCUDHINfzDq5J1BHXHCqufXk9rZubKIuzjgfLE7gCGOn9vgC3lvrGYOXlldUenHPiTt7vJWHEzcMZinFAB5x7u0oaN6AvAau8Mh0BHKS+Ab0dZ2STgHk4pxodvWP5eEYsIP51wCCcc5bLcY5ORjVRjlU4ByjJwC1eGXbzcY/q8g4on2e88rgQqACGBdPpXetx4Hzvd29i4z+goQF9CljUIM4u4HLv+zDcP7Eq73pXgFvq3tTHmvDxxcPAP4lIzzDSfqqqBer68n6N8wm5UFUrVHUdrqYwOCD+71T1XVWtAB4Evi4i/YF/wDWxC9S5RPsr8AowJSDt66r6B1WtUdXTgSK8c4wD5qtrDm0DluM8A7WEenmq6kZV/cD7/Tect6LLm0hfoKofq+opnFH7ahhxp+C2Tdmszj/owzTi7s7rZpgMPKyqZar6IfCLwDiq+ktVLfHK+XHcP7PzGxOlqr9T1T3qeAdXO27KHeBnwH+qaqWq/hpnMK4L8x4tUNVTqrod5zX+wibi1gAjRKSTqh5W1WjvlBCMHGCe13w/JiLHcO9IH89f7Vu4Ckwqbp+xTODfmjupGdA4wnvJ1uCa86FyJOD7Ke98DcPSAn4fCMj3JK4p1wf3oF3c4EG7DeeG7qy0QegDfK6qgQ5x9+GaRS2hXp4icrGI/F5EikWkFNcE69FE+sBtRMqpXxZ+4/ahfrmVE6SvzaMnrmYZqHtfg2uYJyI7vN0oj+H2jGr0GkRkgoj8SUQ+9+JPbCo+cFC96lVA/n0I7x75Kj91fdu34O7HYRH5nYh8pYnzRosDwL+qakbAp7OqvojzdtUfWOpVOEpwXqEmNndSM6DxxyPAXdR/mGsHXAI9iwcatHDoX/tFRNJwD9Eh3IP2ToMHLU1VvxeQtikXXoeAbiISuCXDAFzz1Q+Nnbth+EpcH15/Ve2K69MVn3mEy2HcNh8AiEgnXBM8GMW45mD/gLABAWkvxW0CdzOQqaoZuCZ27TXUu15vq99XcN0GWV78N2n6mvuKSODxAbj709J7FMhZ90tV31LVb+Ka7ztxzf+oICJJItIR15WVKCIdvelIzwBzvH+8IiKpInKdiKSr28nhU+B7XvoMXB/p9sZzcpgBjTNUdTeuCT43IKwY93DfLiKJIjIb1xfWEiaKyDhvf59FwJ9V9QCuBjxURGaISLL3GSMivrax8M7xR+BR7+G9ADd49Cufuo4A/XzsO5SOq0WdFpGvAc3uR98K/AaYJCJ/7+lbQCMGzOtKeRXIF5HOIpKHeylrScfnWRhIAAABsUlEQVQZ2GKcx/mHqb+1xREg12tegutzTfHiV4nIBFzfdlP0AuZ693Aqrp/vzVa4R4HU0ykiWSJyvTclrwLXZxzNKWIP4Vpb9+H6mE8BD6nqVlzFZCnwBW7gbFZAuptwg2LF3rHa/uMmMQManyzE9cUEchfwv3FNxuG4F6AlrMTVdj8HLsI10/GadeNxm6kdwjXd/g338vplGm7w4RCwGnhEVd/2mXYDbpuHIhFpao+nfwQWisgJXF/kqhD0hYXXl/dPuMGhw7iBuM9whiIY38c1dYtwgxIFAcfeAtbituzYh5t9ENjcf9n7WyIif/Xuy1zcdX6B+4fxRjOS/wwMwQ0C/iswRb/cKqMl9yiQejpxNmWed97P8TmfsrVQ1XxVlQaffO/Yf6nqGK9V1VtVp9Z2Y6jqNlW9QlUzVbWHd+yz5vIzh8qGESZe18cxYIiqfhprPYGIyCzcLI1xsdbSnrEaqGGEgIhM8prkqbj+yA9w09CMcxAzoIYRGjfw5UDMEOBWtWbcOYs14Q3DMMLEaqCGYRhhYgbUMAwjTMyAGoZhhIkZUMMwjDAxA2oYhhEm/x8Ws3+jZ2eTXgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 360x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "cmap = cm.get_cmap(\"tab10\")\n",
    "\n",
    "x = df[\"datapoint\"].values\n",
    "accuracy_m_mean, accuracy_m_std = df[\"accuracy_m\"].ewm(com=16, min_periods=0).agg(['mean', 'std']).values.T\n",
    "accuracy_mv_mean, accuracy_mv_std = df[\"accuracy_mv\"].ewm(com=16, min_periods=0).agg(['mean', 'std']).values.T\n",
    "\n",
    "# accuracy_m_std = np.nan_to_num(accuracy_m_std, 0)\n",
    "# accuracy_mv_std = np.nan_to_num(accuracy_mv_std, 0)\n",
    "\n",
    "fg, ax = plt.subplots(figsize=(5, 4))\n",
    "\n",
    "ax.plot(x, accuracy_m_mean, label=\"training\", color=cmap(0))\n",
    "ax.fill_between(x, accuracy_m_mean - accuracy_m_std * 2, accuracy_m_mean + accuracy_m_std * 2, alpha=0.25, color=cmap(0))\n",
    "ax.plot(x, accuracy_mv_mean, label=\"validation\", color=cmap(1))\n",
    "ax.fill_between(x, accuracy_mv_mean - accuracy_mv_std * 2, accuracy_mv_mean + accuracy_mv_std * 2, alpha=0.25, color=cmap(1))\n",
    "\n",
    "# ax.hlines(0.7, 0, df[\"datapoint\"].max(), linestyle='--')\n",
    "ax.legend(loc=\"lower right\")\n",
    "ax.set_xlabel(\"Number of training data points\")\n",
    "ax.set_ylabel(\"Accuracy on missing residues\")\n",
    "ax.yaxis.set_major_formatter(FormatStrFormatter('%.1f'))\n",
    "\n",
    "# ax.set_ylim(0, 0.82)\n",
    "# fg.savefig(NOTEBOOK_PATH / f\"{UNIQUE_ID}-training-curve.svg\", bbox_inches=\"tight\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test set sequence identity"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Oneshot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/strokach/env/lib/python3.7/site-packages/ipykernel_launcher.py:4: TqdmDeprecationWarning: This function will be removed in tqdm==5.0.0\n",
      "Please use `tqdm.notebook.tqdm` instead of `tqdm.tqdm_notebook`\n",
      "  after removing the cwd from sys.path.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4dfb72e15a204da699b049d15cf9b01d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=1283.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "results = []\n",
    "for i, data in enumerate(\n",
    "    tqdm.tqdm_notebook(\n",
    "        DataLoader(datasets[\"protein_test\"], shuffle=False, num_workers=1, batch_size=1, drop_last=False)\n",
    "    )\n",
    "):\n",
    "    data = data.to(device)\n",
    "    data.y = data.x\n",
    "    x_in = torch.ones_like(data.y) * 20\n",
    "    is_missing = torch.ones(data.y.size(0), dtype=torch.bool)\n",
    "    output = net(x_in, data.edge_index, data.edge_attr)\n",
    "    output = torch.softmax(output, dim=1)\n",
    "    _, predicted = output.max(dim=1)\n",
    "    num_correct = float((predicted[is_missing] == data.y[is_missing]).sum())\n",
    "    num_total = float(is_missing.sum())\n",
    "    results.append({\"fraction_correct\": num_correct / num_total, \"num_correct\": num_correct, \"num_total\": num_total})\n",
    "oneshot_results_df = pd.DataFrame(results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.2728685383656129"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fraction_correct_oneshot = (oneshot_results_df[\"num_correct\"] / oneshot_results_df[\"num_total\"]).mean()\n",
    "fraction_correct_oneshot"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Incremental"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "@torch.no_grad()\n",
    "def design_protein(net, x, edge_index, edge_attr):\n",
    "    mask = (x == 20)\n",
    "    if not mask.any():\n",
    "        return x\n",
    "    output = net(x, edge_index, edge_attr)\n",
    "    output = torch.softmax(output, dim=1)\n",
    "    max_pred, max_index = output.max(dim=1)\n",
    "    max_pred[~mask] = -1\n",
    "    _, max_residue = max_pred.max(dim=0)\n",
    "    x[max_residue] = max_index[max_residue]\n",
    "    return design_protein(net, x, edge_index, edge_attr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_sequence_identity = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for frac_present in [0.0, 0.5, 0.8]:\n",
    "    results = []\n",
    "    for i, data in enumerate(\n",
    "        tqdm.tqdm_notebook(\n",
    "            DataLoader(datasets[\"protein_test\"], shuffle=False, num_workers=1, batch_size=1, drop_last=False)\n",
    "        )\n",
    "    ):\n",
    "        data.to(device)\n",
    "        if frac_present == 0:\n",
    "            is_present = torch.zeros(data.x.size(0), dtype=torch.bool, device=data.x.device)\n",
    "            x_in = torch.ones_like(data.x) * 20\n",
    "        else:\n",
    "            is_present = torch.rand(data.x.size(0), device=data.x.device) < frac_present\n",
    "            x_in = torch.where(is_present, data.x, torch.tensor(20, device=data.x.device))\n",
    "        x_pred = design_protein(net, x_in, data.edge_index, data.edge_attr)\n",
    "        identity_all = float((x_pred == data.x).sum()) / data.x.size(0)\n",
    "        identity_missing = float((x_pred[~is_present] == data.x[~is_present]).sum()) / (~is_present).sum().item()\n",
    "        result = {\n",
    "            \"identity_all\": identity_all,\n",
    "            \"identity_missing\": identity_missing,\n",
    "            \"num_correct\": (x_pred[~is_present] == data.x[~is_present]).sum().item(),\n",
    "            \"num_total\": (~is_present).sum().item(),\n",
    "        }\n",
    "        results.append(result)\n",
    "    test_sequence_identity[frac_present] = pd.DataFrame(results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fraction_correct_incremental = (test_sequence_identity[0.0][\"num_correct\"] / test_sequence_identity[0.0][\"num_total\"]).mean()\n",
    "fraction_correct_incremental"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "xlim = -0.025, 0.625\n",
    "\n",
    "fg, ax = plt.subplots(figsize=(5, 4))\n",
    "for frac, df in sorted(test_sequence_identity.items()):\n",
    "    data = df[\"identity_missing\"]\n",
    "    data = np.clip(data, *xlim)\n",
    "#         ax.hist(data, bins=30, range=xlim, histtype=\"step\", label=f\"{frac:.0%} missing residues\", linewidth=1.5)\n",
    "    sns.kdeplot(data, gridsize=35, label=f\"{frac:.0%} residues present\")\n",
    "ax.set_ylabel(\"Number of sequences (normalized)\")\n",
    "ax.set_xlabel(\"Sequence identity to reference\")\n",
    "ax.set_xlim(*xlim)\n",
    "# ax.set_ylim(0, 4.2)\n",
    "ax.legend(loc=\"upper left\", framealpha=0.3)\n",
    "fg.tight_layout()\n",
    "fg.savefig(NOTEBOOK_PATH.joinpath(f\"{UNIQUE_ID}-test-incremental-kde.svg\"), bbox_inches=\"tight\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fg, ax = plt.subplots(figsize=(5, 4))\n",
    "\n",
    "xlim = -0.025, 0.625\n",
    "\n",
    "data = oneshot_results_df[\"fraction_correct\"]\n",
    "data = np.clip(data, *xlim)\n",
    "ax.hist(\n",
    "    data,\n",
    "    bins=100,\n",
    "    range=xlim,\n",
    "    alpha=0.9,\n",
    "    label=f\"oneshot (accuracy: {fraction_correct_oneshot:.2%})\",\n",
    "    histtype=\"stepfilled\",\n",
    "    linewidth=2,\n",
    "    color=cm.get_cmap(\"Set1\")(1),\n",
    ")\n",
    "# sns.kdeplot(data, gridsize=200, label=\"oneshot\")\n",
    "\n",
    "data = test_sequence_identity[0.0][\"identity_missing\"]\n",
    "data = np.clip(data, *xlim)\n",
    "ax.hist(\n",
    "    data,\n",
    "    bins=100,\n",
    "    range=xlim,\n",
    "    alpha=0.6,\n",
    "    label=f\"incremental (accuracy: {fraction_correct_incremental:.2%})\",\n",
    "    histtype=\"stepfilled\",\n",
    "    linewidth=2,\n",
    "    color=cm.get_cmap(\"Set1\")(0),\n",
    ")\n",
    "# sns.kdeplot(data, gridsize=100, label=\"incremental\")\n",
    "\n",
    "ax.set_ylabel(\"Number of sequences\")\n",
    "ax.set_xlabel(\"Sequence identity to reference\")\n",
    "ax.set_xlim(xlim)\n",
    "# ax.set_ylim(0, 165)\n",
    "\n",
    "ax.legend(framealpha=0.5)\n",
    "\n",
    "fg.tight_layout()\n",
    "fg.savefig(NOTEBOOK_PATH.joinpath(f\"{UNIQUE_ID}-test-oneshot-incremental.svg\"), bbox_inches=\"tight\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Protherm validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "results = []\n",
    "\n",
    "for i, data in enumerate(tqdm.tqdm_notebook(protherm_dataset, total=len(protherm_dataset))):\n",
    "    data = data.to(device)\n",
    "\n",
    "    row = protherm_df.loc[data.Index.item()]\n",
    "    with torch.no_grad():\n",
    "        output = net(data.x, data.edge_index, data.edge_attr)\n",
    "\n",
    "    wt_idx = torch.tensor(proteinsolver.utils.seq_to_tensor(row.mutation[0].encode()), dtype=torch.long, device=device)\n",
    "    mut_idx = torch.tensor(\n",
    "        proteinsolver.utils.seq_to_tensor(row.mutation[-1].encode()), dtype=torch.long, device=device\n",
    "    )\n",
    "    pos = int(row.mutation[1:-1])\n",
    "    x_mut = data.x.clone()\n",
    "    assert x_mut[pos - 1] == wt_idx\n",
    "    x_mut[pos - 1] = mut_idx\n",
    "    with torch.no_grad():\n",
    "        output_mut = net(x_mut, data.edge_index, data.edge_attr)\n",
    "\n",
    "    x_blank = data.x.clone()\n",
    "    assert x_blank[pos - 1] == wt_idx\n",
    "    x_blank[pos - 1] = 20\n",
    "    with torch.no_grad():\n",
    "        output_blank = net(x_blank, data.edge_index, data.edge_attr)\n",
    "\n",
    "    # logproba.scan\n",
    "    scan_wt = proteinsolver.utils.scan_with_mask(net, data.x, data.edge_index, data.edge_attr)\n",
    "    scan_sm_wt = proteinsolver.utils.scan_with_mask(\n",
    "        net, data.x, data.edge_index, data.edge_attr, output_transform=\"proba\"\n",
    "    )\n",
    "    scan_sm_log_wt = proteinsolver.utils.scan_with_mask(\n",
    "        net, data.x, data.edge_index, data.edge_attr, output_transform=\"logproba\"\n",
    "    )\n",
    "    scan_mut = proteinsolver.utils.scan_with_mask(net, x_mut, data.edge_index, data.edge_attr)\n",
    "    scan_sm_mut = proteinsolver.utils.scan_with_mask(\n",
    "        net, x_mut, data.edge_index, data.edge_attr, output_transform=\"proba\"\n",
    "    )\n",
    "    scan_sm_log_mut = proteinsolver.utils.scan_with_mask(\n",
    "        net, x_mut, data.edge_index, data.edge_attr, output_transform=\"logproba\"\n",
    "    )\n",
    "\n",
    "    # logproba.incremental\n",
    "    incremental_wt = proteinsolver.utils.get_node_outputs(net, data.x, data.edge_index, data.edge_attr)\n",
    "    incremental_sm_wt = proteinsolver.utils.get_node_outputs(\n",
    "        net, data.x, data.edge_index, data.edge_attr, output_transform=\"proba\"\n",
    "    )\n",
    "    incremental_sm_log_wt = proteinsolver.utils.get_node_outputs(\n",
    "        net, data.x, data.edge_index, data.edge_attr, output_transform=\"logproba\"\n",
    "    )\n",
    "    incremental_mut = proteinsolver.utils.get_node_outputs(net, x_mut, data.edge_index, data.edge_attr)\n",
    "    incremental_sm_mut = proteinsolver.utils.get_node_outputs(\n",
    "        net, x_mut, data.edge_index, data.edge_attr, output_transform=\"proba\"\n",
    "    )\n",
    "    incremental_sm_log_mut = proteinsolver.utils.get_node_outputs(\n",
    "        net, x_mut, data.edge_index, data.edge_attr, output_transform=\"logproba\"\n",
    "    )\n",
    "\n",
    "    result = {\n",
    "        \"wt_wt\": output[pos - 1, wt_idx].item(),\n",
    "        \"wt_mut\": output[pos - 1, mut_idx].item(),\n",
    "        \"wt_sm_wt\": torch.softmax(output, dim=1)[pos - 1, wt_idx].item(),\n",
    "        \"wt_sm_mut\": torch.softmax(output, dim=1)[pos - 1, mut_idx].item(),\n",
    "        \"wt_sm_log_wt\": torch.softmax(output, dim=1).log()[pos - 1, wt_idx].item(),\n",
    "        \"wt_sm_log_mut\": torch.softmax(output, dim=1).log()[pos - 1, mut_idx].item(),\n",
    "        #\n",
    "        \"mut_wt\": output_mut[pos - 1, wt_idx].item(),\n",
    "        \"mut_mut\": output_mut[pos - 1, mut_idx].item(),\n",
    "        \"mut_sm_wt\": torch.softmax(output_mut, dim=1)[pos - 1, wt_idx].item(),\n",
    "        \"mut_sm_mut\": torch.softmax(output_mut, dim=1)[pos - 1, mut_idx].item(),\n",
    "        \"mut_sm_log_wt\": torch.softmax(output_mut, dim=1).log()[pos - 1, wt_idx].item(),\n",
    "        \"mut_sm_log_mut\": torch.softmax(output_mut, dim=1).log()[pos - 1, mut_idx].item(),\n",
    "        #\n",
    "        \"blank_wt\": output_blank[pos - 1, wt_idx].item(),\n",
    "        \"blank_mut\": output_blank[pos - 1, mut_idx].item(),\n",
    "        \"blank_sm_wt\": torch.softmax(output_blank, dim=1)[pos - 1, wt_idx].item(),\n",
    "        \"blank_sm_mut\": torch.softmax(output_blank, dim=1)[pos - 1, mut_idx].item(),\n",
    "        \"blank_sm_log_wt\": torch.softmax(output_blank, dim=1).log()[pos - 1, wt_idx].item(),\n",
    "        \"blank_sm_log_mut\": torch.softmax(output_blank, dim=1).log()[pos - 1, mut_idx].item(),\n",
    "        #\n",
    "        \"scan_wt\": scan_wt.mean().item(),\n",
    "        \"scan_mut\": scan_mut.mean().item(),\n",
    "        \"scan_sm_wt\": scan_sm_wt.mean().item(),\n",
    "        \"scan_sm_mut\": scan_sm_mut.mean().item(),\n",
    "        \"scan_sm_log_wt\": scan_sm_log_wt.mean().item(),\n",
    "        \"scan_sm_log_mut\": scan_sm_log_mut.mean().item(),\n",
    "        #\n",
    "        \"incremental_wt\": incremental_wt.mean().item(),\n",
    "        \"incremental_mut\": incremental_mut.mean().item(),\n",
    "        \"incremental_sm_wt\": incremental_sm_wt.mean().item(),\n",
    "        \"incremental_sm_mut\": incremental_sm_mut.mean().item(),\n",
    "        \"incremental_sm_log_wt\": incremental_sm_log_wt.mean().item(),\n",
    "        \"incremental_sm_log_mut\": incremental_sm_log_mut.mean().item(),\n",
    "        #\n",
    "        \"ddg_exp\": row.ddg_exp,\n",
    "    }\n",
    "    results.append(result)\n",
    "\n",
    "results_df = pd.DataFrame(results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'results_df' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-24-dbef2ba383d4>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresults_df\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'results_df' is not defined"
     ]
    }
   ],
   "source": [
    "len(protherm_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "# torch.save(results_df, NOTEBOOK_PATH.joinpath(\"protherm_results.torch\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_df = torch.load(NOTEBOOK_PATH.joinpath(\"protherm_results.torch\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>filename_wt</th>\n",
       "      <th>chain_id</th>\n",
       "      <th>mutation</th>\n",
       "      <th>cartesian_ddg_beta_nov15_cart_1</th>\n",
       "      <th>ddg_exp</th>\n",
       "      <th>cartesian_ddg_beta_nov16_cart_1</th>\n",
       "      <th>cartesian_ddg_score12_cart_1</th>\n",
       "      <th>cartesian_ddg_talaris2013_cart_1</th>\n",
       "      <th>cartesian_ddg_talaris2014_cart_1</th>\n",
       "      <th>ddg_monomer_soft_rep_design_1</th>\n",
       "      <th>local_filename_wt</th>\n",
       "      <th>structure_id</th>\n",
       "      <th>model_id</th>\n",
       "      <th>qseq</th>\n",
       "      <th>residue_idx_1_corrected</th>\n",
       "      <th>residue_idx_2_corrected</th>\n",
       "      <th>distances</th>\n",
       "      <th>mutation_matches_sequence</th>\n",
       "      <th>qseq_mutation</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>/home/kimlab2/database_data/biological-data-wa...</td>\n",
       "      <td>A</td>\n",
       "      <td>G44S</td>\n",
       "      <td>-1.808667</td>\n",
       "      <td>-0.53</td>\n",
       "      <td>-0.701000</td>\n",
       "      <td>0.088000</td>\n",
       "      <td>-0.289667</td>\n",
       "      <td>-0.633667</td>\n",
       "      <td>-2.384</td>\n",
       "      <td>/home/kimlab1/database_data/datapkg/adjacency-...</td>\n",
       "      <td>107l</td>\n",
       "      <td>0</td>\n",
       "      <td>MNIFEMLRIDEGLRLKIYKDTEGYYTIGIGHLLTKSPSLNAAKGEL...</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "      <td>[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 66, 70, 91...</td>\n",
       "      <td>[1.3463744749991646, 4.7287436401778065, 6.389...</td>\n",
       "      <td>True</td>\n",
       "      <td>MNIFEMLRIDEGLRLKIYKDTEGYYTIGIGHLLTKSPSLNAAKSEL...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>/home/kimlab2/database_data/biological-data-wa...</td>\n",
       "      <td>A</td>\n",
       "      <td>A120M</td>\n",
       "      <td>2.617667</td>\n",
       "      <td>-0.20</td>\n",
       "      <td>0.354000</td>\n",
       "      <td>0.560000</td>\n",
       "      <td>-0.069000</td>\n",
       "      <td>-0.188000</td>\n",
       "      <td>2.472</td>\n",
       "      <td>/home/kimlab1/database_data/datapkg/adjacency-...</td>\n",
       "      <td>160l</td>\n",
       "      <td>0</td>\n",
       "      <td>MNIFEMLRIDEGLRLKIYKDTEGYYTIGIGHLLTKSPSLNAAKSEL...</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "      <td>[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 66, 70, 91, 92...</td>\n",
       "      <td>[1.345760255620924, 4.727237880524551, 6.42311...</td>\n",
       "      <td>True</td>\n",
       "      <td>MNIFEMLRIDEGLRLKIYKDTEGYYTIGIGHLLTKSPSLNAAKSEL...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>/home/kimlab2/database_data/biological-data-wa...</td>\n",
       "      <td>A</td>\n",
       "      <td>A116N</td>\n",
       "      <td>0.502667</td>\n",
       "      <td>0.17</td>\n",
       "      <td>-0.131333</td>\n",
       "      <td>0.419000</td>\n",
       "      <td>0.392000</td>\n",
       "      <td>0.114000</td>\n",
       "      <td>0.652</td>\n",
       "      <td>/home/kimlab1/database_data/datapkg/adjacency-...</td>\n",
       "      <td>161l</td>\n",
       "      <td>0</td>\n",
       "      <td>MNIFEMLRIDEGLRLKIYKDTEGYYTIGIGHLLTKSPSLNAAKSEL...</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "      <td>[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 28, 66, 70...</td>\n",
       "      <td>[1.3497327388224938, 4.72480284033153, 6.38761...</td>\n",
       "      <td>True</td>\n",
       "      <td>MNIFEMLRIDEGLRLKIYKDTEGYYTIGIGHLLTKSPSLNAAKSEL...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>/home/kimlab2/database_data/biological-data-wa...</td>\n",
       "      <td>A</td>\n",
       "      <td>A122Q</td>\n",
       "      <td>1.760333</td>\n",
       "      <td>-0.24</td>\n",
       "      <td>1.999000</td>\n",
       "      <td>3.205667</td>\n",
       "      <td>0.605667</td>\n",
       "      <td>0.508667</td>\n",
       "      <td>12.540</td>\n",
       "      <td>/home/kimlab1/database_data/datapkg/adjacency-...</td>\n",
       "      <td>162l</td>\n",
       "      <td>0</td>\n",
       "      <td>MNIFEMLRIDEGLRLKIYKDTEGYYTIGIGHLLTKSPSLNAAKSEL...</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "      <td>[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 66, 70, 91...</td>\n",
       "      <td>[1.3505151767500694, 4.763315132476543, 6.4785...</td>\n",
       "      <td>True</td>\n",
       "      <td>MNIFEMLRIDEGLRLKIYKDTEGYYTIGIGHLLTKSPSLNAAKSEL...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>/home/kimlab2/database_data/biological-data-wa...</td>\n",
       "      <td>A</td>\n",
       "      <td>A123Q</td>\n",
       "      <td>0.110333</td>\n",
       "      <td>-0.22</td>\n",
       "      <td>-1.153000</td>\n",
       "      <td>1.313000</td>\n",
       "      <td>-0.334333</td>\n",
       "      <td>-0.577667</td>\n",
       "      <td>-3.730</td>\n",
       "      <td>/home/kimlab1/database_data/datapkg/adjacency-...</td>\n",
       "      <td>163l</td>\n",
       "      <td>0</td>\n",
       "      <td>MNIFEMLRIDEGLRLKIYKDTEGYYTIGIGHLLTKSPSLNAAKSEL...</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "      <td>[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 66, 70, 91, 92...</td>\n",
       "      <td>[1.3461430876707972, 4.734742373580991, 6.4045...</td>\n",
       "      <td>True</td>\n",
       "      <td>MNIFEMLRIDEGLRLKIYKDTEGYYTIGIGHLLTKSPSLNAAKSEL...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                         filename_wt chain_id mutation  \\\n",
       "0  /home/kimlab2/database_data/biological-data-wa...        A     G44S   \n",
       "1  /home/kimlab2/database_data/biological-data-wa...        A    A120M   \n",
       "2  /home/kimlab2/database_data/biological-data-wa...        A    A116N   \n",
       "3  /home/kimlab2/database_data/biological-data-wa...        A    A122Q   \n",
       "4  /home/kimlab2/database_data/biological-data-wa...        A    A123Q   \n",
       "\n",
       "   cartesian_ddg_beta_nov15_cart_1  ddg_exp  cartesian_ddg_beta_nov16_cart_1  \\\n",
       "0                        -1.808667    -0.53                        -0.701000   \n",
       "1                         2.617667    -0.20                         0.354000   \n",
       "2                         0.502667     0.17                        -0.131333   \n",
       "3                         1.760333    -0.24                         1.999000   \n",
       "4                         0.110333    -0.22                        -1.153000   \n",
       "\n",
       "   cartesian_ddg_score12_cart_1  cartesian_ddg_talaris2013_cart_1  \\\n",
       "0                      0.088000                         -0.289667   \n",
       "1                      0.560000                         -0.069000   \n",
       "2                      0.419000                          0.392000   \n",
       "3                      3.205667                          0.605667   \n",
       "4                      1.313000                         -0.334333   \n",
       "\n",
       "   cartesian_ddg_talaris2014_cart_1  ddg_monomer_soft_rep_design_1  \\\n",
       "0                         -0.633667                         -2.384   \n",
       "1                         -0.188000                          2.472   \n",
       "2                          0.114000                          0.652   \n",
       "3                          0.508667                         12.540   \n",
       "4                         -0.577667                         -3.730   \n",
       "\n",
       "                                   local_filename_wt structure_id  model_id  \\\n",
       "0  /home/kimlab1/database_data/datapkg/adjacency-...         107l         0   \n",
       "1  /home/kimlab1/database_data/datapkg/adjacency-...         160l         0   \n",
       "2  /home/kimlab1/database_data/datapkg/adjacency-...         161l         0   \n",
       "3  /home/kimlab1/database_data/datapkg/adjacency-...         162l         0   \n",
       "4  /home/kimlab1/database_data/datapkg/adjacency-...         163l         0   \n",
       "\n",
       "                                                qseq  \\\n",
       "0  MNIFEMLRIDEGLRLKIYKDTEGYYTIGIGHLLTKSPSLNAAKGEL...   \n",
       "1  MNIFEMLRIDEGLRLKIYKDTEGYYTIGIGHLLTKSPSLNAAKSEL...   \n",
       "2  MNIFEMLRIDEGLRLKIYKDTEGYYTIGIGHLLTKSPSLNAAKSEL...   \n",
       "3  MNIFEMLRIDEGLRLKIYKDTEGYYTIGIGHLLTKSPSLNAAKSEL...   \n",
       "4  MNIFEMLRIDEGLRLKIYKDTEGYYTIGIGHLLTKSPSLNAAKSEL...   \n",
       "\n",
       "                             residue_idx_1_corrected  \\\n",
       "0  [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...   \n",
       "1  [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...   \n",
       "2  [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...   \n",
       "3  [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...   \n",
       "4  [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...   \n",
       "\n",
       "                             residue_idx_2_corrected  \\\n",
       "0  [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 66, 70, 91...   \n",
       "1  [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 66, 70, 91, 92...   \n",
       "2  [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 28, 66, 70...   \n",
       "3  [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 66, 70, 91...   \n",
       "4  [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 66, 70, 91, 92...   \n",
       "\n",
       "                                           distances  \\\n",
       "0  [1.3463744749991646, 4.7287436401778065, 6.389...   \n",
       "1  [1.345760255620924, 4.727237880524551, 6.42311...   \n",
       "2  [1.3497327388224938, 4.72480284033153, 6.38761...   \n",
       "3  [1.3505151767500694, 4.763315132476543, 6.4785...   \n",
       "4  [1.3461430876707972, 4.734742373580991, 6.4045...   \n",
       "\n",
       "   mutation_matches_sequence  \\\n",
       "0                       True   \n",
       "1                       True   \n",
       "2                       True   \n",
       "3                       True   \n",
       "4                       True   \n",
       "\n",
       "                                       qseq_mutation  \n",
       "0  MNIFEMLRIDEGLRLKIYKDTEGYYTIGIGHLLTKSPSLNAAKSEL...  \n",
       "1  MNIFEMLRIDEGLRLKIYKDTEGYYTIGIGHLLTKSPSLNAAKSEL...  \n",
       "2  MNIFEMLRIDEGLRLKIYKDTEGYYTIGIGHLLTKSPSLNAAKSEL...  \n",
       "3  MNIFEMLRIDEGLRLKIYKDTEGYYTIGIGHLLTKSPSLNAAKSEL...  \n",
       "4  MNIFEMLRIDEGLRLKIYKDTEGYYTIGIGHLLTKSPSLNAAKSEL...  "
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "protherm_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert len(results_df) == len(protherm_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert (results_df.index == protherm_df.index).all()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "protherm_wresults_df = pd.concat([protherm_df, results_df.rename(columns={\"ddg_exp\": \"ddg_exp_2\"})], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert (protherm_wresults_df[\"ddg_exp\"] == protherm_wresults_df[\"ddg_exp_2\"]).all()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "del protherm_wresults_df[\"ddg_exp_2\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>filename_wt</th>\n",
       "      <th>chain_id</th>\n",
       "      <th>mutation</th>\n",
       "      <th>cartesian_ddg_beta_nov15_cart_1</th>\n",
       "      <th>ddg_exp</th>\n",
       "      <th>cartesian_ddg_beta_nov16_cart_1</th>\n",
       "      <th>cartesian_ddg_score12_cart_1</th>\n",
       "      <th>cartesian_ddg_talaris2013_cart_1</th>\n",
       "      <th>cartesian_ddg_talaris2014_cart_1</th>\n",
       "      <th>ddg_monomer_soft_rep_design_1</th>\n",
       "      <th>...</th>\n",
       "      <th>blank_change</th>\n",
       "      <th>blank_sm_change</th>\n",
       "      <th>wt_mut_change</th>\n",
       "      <th>wt_mut_sm_change</th>\n",
       "      <th>scan_change</th>\n",
       "      <th>scan_sm_change</th>\n",
       "      <th>scan_sm_log_change</th>\n",
       "      <th>incremental_change</th>\n",
       "      <th>incremental_sm_change</th>\n",
       "      <th>incremental_sm_log_change</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>/home/kimlab2/database_data/biological-data-wa...</td>\n",
       "      <td>A</td>\n",
       "      <td>G44S</td>\n",
       "      <td>-1.808667</td>\n",
       "      <td>-0.53</td>\n",
       "      <td>-0.701000</td>\n",
       "      <td>0.088000</td>\n",
       "      <td>-0.289667</td>\n",
       "      <td>-0.633667</td>\n",
       "      <td>-2.384</td>\n",
       "      <td>...</td>\n",
       "      <td>0.973255</td>\n",
       "      <td>0.022096</td>\n",
       "      <td>-5.575768</td>\n",
       "      <td>-0.000003</td>\n",
       "      <td>0.012084</td>\n",
       "      <td>0.000869</td>\n",
       "      <td>0.010134</td>\n",
       "      <td>0.006004</td>\n",
       "      <td>0.000135</td>\n",
       "      <td>0.005919</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>/home/kimlab2/database_data/biological-data-wa...</td>\n",
       "      <td>A</td>\n",
       "      <td>A120M</td>\n",
       "      <td>2.617667</td>\n",
       "      <td>-0.20</td>\n",
       "      <td>0.354000</td>\n",
       "      <td>0.560000</td>\n",
       "      <td>-0.069000</td>\n",
       "      <td>-0.188000</td>\n",
       "      <td>2.472</td>\n",
       "      <td>...</td>\n",
       "      <td>-2.342890</td>\n",
       "      <td>-0.238818</td>\n",
       "      <td>2.544128</td>\n",
       "      <td>-0.000006</td>\n",
       "      <td>-0.028014</td>\n",
       "      <td>-0.001755</td>\n",
       "      <td>-0.012493</td>\n",
       "      <td>-0.022250</td>\n",
       "      <td>-0.001574</td>\n",
       "      <td>-0.015689</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>/home/kimlab2/database_data/biological-data-wa...</td>\n",
       "      <td>A</td>\n",
       "      <td>A116N</td>\n",
       "      <td>0.502667</td>\n",
       "      <td>0.17</td>\n",
       "      <td>-0.131333</td>\n",
       "      <td>0.419000</td>\n",
       "      <td>0.392000</td>\n",
       "      <td>0.114000</td>\n",
       "      <td>0.652</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.437747</td>\n",
       "      <td>-0.083779</td>\n",
       "      <td>-0.744164</td>\n",
       "      <td>-0.000010</td>\n",
       "      <td>-0.025267</td>\n",
       "      <td>-0.001827</td>\n",
       "      <td>-0.008965</td>\n",
       "      <td>-0.014738</td>\n",
       "      <td>-0.000584</td>\n",
       "      <td>-0.007784</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>/home/kimlab2/database_data/biological-data-wa...</td>\n",
       "      <td>A</td>\n",
       "      <td>A122Q</td>\n",
       "      <td>1.760333</td>\n",
       "      <td>-0.24</td>\n",
       "      <td>1.999000</td>\n",
       "      <td>3.205667</td>\n",
       "      <td>0.605667</td>\n",
       "      <td>0.508667</td>\n",
       "      <td>12.540</td>\n",
       "      <td>...</td>\n",
       "      <td>-5.290084</td>\n",
       "      <td>-0.741825</td>\n",
       "      <td>0.841757</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>-0.085311</td>\n",
       "      <td>-0.005899</td>\n",
       "      <td>-0.029683</td>\n",
       "      <td>-0.038967</td>\n",
       "      <td>-0.004397</td>\n",
       "      <td>-0.028937</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>/home/kimlab2/database_data/biological-data-wa...</td>\n",
       "      <td>A</td>\n",
       "      <td>A123Q</td>\n",
       "      <td>0.110333</td>\n",
       "      <td>-0.22</td>\n",
       "      <td>-1.153000</td>\n",
       "      <td>1.313000</td>\n",
       "      <td>-0.334333</td>\n",
       "      <td>-0.577667</td>\n",
       "      <td>-3.730</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.963735</td>\n",
       "      <td>-0.128742</td>\n",
       "      <td>0.783663</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>-0.012075</td>\n",
       "      <td>-0.001521</td>\n",
       "      <td>-0.002958</td>\n",
       "      <td>-0.001591</td>\n",
       "      <td>-0.001283</td>\n",
       "      <td>-0.002547</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 59 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                         filename_wt chain_id mutation  \\\n",
       "0  /home/kimlab2/database_data/biological-data-wa...        A     G44S   \n",
       "1  /home/kimlab2/database_data/biological-data-wa...        A    A120M   \n",
       "2  /home/kimlab2/database_data/biological-data-wa...        A    A116N   \n",
       "3  /home/kimlab2/database_data/biological-data-wa...        A    A122Q   \n",
       "4  /home/kimlab2/database_data/biological-data-wa...        A    A123Q   \n",
       "\n",
       "   cartesian_ddg_beta_nov15_cart_1  ddg_exp  cartesian_ddg_beta_nov16_cart_1  \\\n",
       "0                        -1.808667    -0.53                        -0.701000   \n",
       "1                         2.617667    -0.20                         0.354000   \n",
       "2                         0.502667     0.17                        -0.131333   \n",
       "3                         1.760333    -0.24                         1.999000   \n",
       "4                         0.110333    -0.22                        -1.153000   \n",
       "\n",
       "   cartesian_ddg_score12_cart_1  cartesian_ddg_talaris2013_cart_1  \\\n",
       "0                      0.088000                         -0.289667   \n",
       "1                      0.560000                         -0.069000   \n",
       "2                      0.419000                          0.392000   \n",
       "3                      3.205667                          0.605667   \n",
       "4                      1.313000                         -0.334333   \n",
       "\n",
       "   cartesian_ddg_talaris2014_cart_1  ddg_monomer_soft_rep_design_1  ...  \\\n",
       "0                         -0.633667                         -2.384  ...   \n",
       "1                         -0.188000                          2.472  ...   \n",
       "2                          0.114000                          0.652  ...   \n",
       "3                          0.508667                         12.540  ...   \n",
       "4                         -0.577667                         -3.730  ...   \n",
       "\n",
       "  blank_change blank_sm_change  wt_mut_change wt_mut_sm_change scan_change  \\\n",
       "0     0.973255        0.022096      -5.575768        -0.000003    0.012084   \n",
       "1    -2.342890       -0.238818       2.544128        -0.000006   -0.028014   \n",
       "2    -1.437747       -0.083779      -0.744164        -0.000010   -0.025267   \n",
       "3    -5.290084       -0.741825       0.841757         0.000002   -0.085311   \n",
       "4    -0.963735       -0.128742       0.783663         0.000002   -0.012075   \n",
       "\n",
       "  scan_sm_change scan_sm_log_change  incremental_change incremental_sm_change  \\\n",
       "0       0.000869           0.010134            0.006004              0.000135   \n",
       "1      -0.001755          -0.012493           -0.022250             -0.001574   \n",
       "2      -0.001827          -0.008965           -0.014738             -0.000584   \n",
       "3      -0.005899          -0.029683           -0.038967             -0.004397   \n",
       "4      -0.001521          -0.002958           -0.001591             -0.001283   \n",
       "\n",
       "   incremental_sm_log_change  \n",
       "0                   0.005919  \n",
       "1                  -0.015689  \n",
       "2                  -0.007784  \n",
       "3                  -0.028937  \n",
       "4                  -0.002547  \n",
       "\n",
       "[5 rows x 59 columns]"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "protherm_wresults_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "# torch.save(protherm_wresults_df, NOTEBOOK_PATH.joinpath(\"protherm_wresults.torch\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_df[\"scan_sm_log_wt\"] = results_df_bak[\"scan_sm_log_wt\"]\n",
    "results_df[\"scan_sm_log_mut\"] = results_df_bak[\"scan_sm_log_mut\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_df[\"blank_change\"] = results_df[\"blank_mut\"] - results_df[\"blank_wt\"]\n",
    "results_df[\"blank_sm_change\"] = results_df[\"blank_sm_mut\"] - results_df[\"blank_sm_wt\"]\n",
    "\n",
    "results_df[\"wt_mut_change\"] = results_df[\"mut_mut\"] - results_df[\"wt_wt\"]\n",
    "results_df[\"wt_mut_sm_change\"] = results_df[\"mut_sm_mut\"] - results_df[\"wt_sm_wt\"]\n",
    "\n",
    "for prefix in [\"scan\", \"scan_sm\", \"scan_sm_log\", \"incremental\", \"incremental_sm\", \"incremental_sm_log\"]:\n",
    "    results_df[f\"{prefix}_change\"] = results_df[f\"{prefix}_mut\"] - results_df[f\"{prefix}_wt\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaQAAAE4CAYAAAAU6IcNAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nOx9eXxTZdb/92nTJV3pQulCoSA7yia8ddxAR1EEl0GccQV8FWVU5sXXHRdEHEQZlfHHLC/gqIyCM8I4IuACjCKIlH1fSsG2oQVKt3RP2vT5/XF4cm9ub9KkTdKkfb6fTz9pcrfn3tyc7z3nfJ9zGOccEhISEhISHY2Qjh6AhISEhIQEIAlJQkJCQiJAIAlJQkJCQiIgIAlJQkJCQiIgIAlJQkJCQiIgIAlJQkJCQiIgYOjoAXgDycnJPCsrq6OHISEhISHRCvbs2VPKOe+ut6xTEFJWVhZ2797d0cOQkJCQkGgFjLECZ8tkyE5CQkJCIiAgCUlCQkJCIiAgCUlCQkJCIiAgCUlCQkJCIiAgCUlCQkJCIiDgd0JijD3BGNvNGLMwxj5UfZ7FGOOMsRrV38v+Hp+EhISERMegI2TfxQBeB3ATAKPO8m6c8yb/DklCQkJCoqPhd0LinP8LABhjowH09OexN23ahPr6en8eUkIioGE0GnHDDTd09DAkJAAE5sTYAsYYB7ARwDOc81Jv7bi+vh633nqrt3YnIRH0+PLLLzt6CBISdgSSqKEUwBgAvQFcDiAWwCfOVmaMPXIxF7X7woULfhqihISEhISvEDCExDmv4Zzv5pw3cc7PA3gCwHjGWJyT9Zdyzkdzzkd3765bFklCQkJCIogQMISkA37xlXXoKCQCBkePHsXo0aM7ehgdAovFgkGDBqGkpKSjhyIh4TN0hOzbwBiLBBAKIJQxFnnxs2zG2EDGWAhjLAnAewC+55ybfTmerKwsGI1GxMTEoEePHnjwwQdRU1Pjy0P6FOPGjUNkZCRiYmKQnJyMyZMn4+zZs25vzznHc889h6SkJCQlJeHZZ58F57zV7ebNmwfGGDZt2mT/bPHixejbty/i4uKQnp6OJ598Ek1NioAyPz8f1113HaKiojBo0CCHbfXw8ssv4+mnn3b7XNzBJ598gpiYGPtfVFQUGGPYs2dPm/a3ZMkSjB49GhEREZg+fbrDsvz8fDDGHI43f/58h3X27t2La6+91n4//vGPfwQARERE4L//+7/x5ptvtmlcEhJBAc65X/8AvAryftR/rwK4B8DPAGoBnAWwAkCqO/u8/PLLuTtYu3Zti8969+7NN27cyDnn/MyZM3zo0KH8ueeec2t/7qKxsdGr+3OFsWPH8mXLlnHOOa+oqOA33ngjv/fee93e/q9//SsfMGAAN5lM/MyZM3zw4MH8L3/5i8tt8vLy+KWXXsrT0tLs11J8XlFRwTnnvKysjF933XX87bffti+/4oor+JNPPsnr6ur46tWreXx8PC8pKdE9RnFxMU9ISOD19fVun0tb8MEHH/C+ffvy5ubmNm2/Zs0a/vnnn/OZM2fyadOmOSz7+eefOQCn98OFCxd49+7d+ccff8wbGhp4VVUVP3r0qH25yWTiSUlJvKGhoU1j04Peb0JCwpcAsJs7seV+95A4569yzpnm71XO+SrOeR/OeTTnPI1zPpVzfs6fY8vIyMCECRNw+PBhAIDZbMZDDz2EtLQ0ZGRk4KWXXoLNZgMAnDp1Ctdffz2SkpKQnJyM++67D5WVlfZ9ZWVl4c0338SwYcMQHR2NpqYmvPnmm8jIyEBsbCwGDhyIzZs3A6BwzOzZs5Geno709HTMnj0bFosFAPD999+jZ8+eePvtt5GSkoK0tDR88MEHbp1Pt27dcMcdd2D//v1uX4OPPvoITz31FHr27ImMjAw89dRT+PDDD11u88QTT+DNN99EeHi4w+eXXHIJunXrBoAefEJCQpCXlwcAyM3Nxd69ezFv3jwYjUbceeeduOyyy7BmzRrdY2zcuBGjRo1CZGSk/bOsrCy88cYbGDJkCBISEvDggw+ioaHB7XPVw0cffYSpU6eCMYoUWywWPP300+jVqxd69OiBmTNnupw6MHnyZNxxxx1ISkry+NjvvPMObrrpJtx3332IiIhAbGwsBg8ebF/es2dPJCQkYMeOHZ6fmIREECCQc0h+h8lkwoYNGzBy5EgAwLRp02AwGJCXl4d9+/bh22+/xfLlywGQgX3hhRdQXFyMY8eOwWQy4dVXX3XY36pVq7B+/XpUVlbi1KlTWLJkCXbt2oXq6mp88803EE0Ff//732PHjh3Yv38/Dhw4gJ07d+L111+37+fcuXMwm80oKirC+++/j8cffxwVFRWtnk9ZWRn+9a9/oV+/fvbPtm3bZicJPRw5cgTDhw+3vx8+fDiOHDnidP3PPvsM4eHhuOWWW3SXr1y5EnFxcUhOTsaBAwfw6KOP2o/Tt29fxMbGunWsQ4cOYeDAgS0+/+STT/DNN9/g1KlTyM3NtV+3wsJCdOvWzenfypUrW+yroKAAP/zwA6ZOnWr/7LnnnkNubi7279+PvLw8FBUV4bXXXnN6PdxB79690bNnTzz44IMoLVVmNezYsQOJiYm48sorkZKSgltvvRWFhYUO2w4ePBgHDhxo1/ElJAIWzlynYPprb8guOjqax8fH8169evHf/va3vK6ujp87d46Hh4fzuro6+7orV67k48aN0933559/zkeMGOGw3/fff9/+/uTJk7x79+5848aN3Gq1Omzbt29fvn79evv7r7/+mvfu3Ztzzvl3333HIyMjHcI83bt35z/99JPuOMaOHcuNRiOPi4vjAPjw4cN5QUGBi6viiJCQEH7s2DH7+9zcXA5AN4RVXV3N+/Xrx0+fPm0/Z3XITo3c3Fz+0ksv8bNnz3LOOV+xYgXPzs52WGfOnDktwlwCDz/8cItQau/evR3CievXr+d9+/Zt/SSd4LXXXuNjx461v29ubuZRUVE8Ly/P/tn27dt5VlZWq/t68cUXW5xLdXU137VrF29sbOTnzp3jd955Jx8/frx9ef/+/Xl8fDzfuXMnr6+v57NmzeJXXnmlwz7uvfdePm/evLadoA5kyE7C30AghewCEf/+979RWVmJgoIC/PnPf4bRaERBQQEaGxuRlpZmf6p+9NFH7SqnkpIS3H333cjIyEBcXBzuv/9+h6ddAMjMzLT/369fPyxevBivvvoqUlJScPfdd6O4uBgAUFxcjN69e9vX7d27t30ZACQlJcFgUOYwR0VFuRRevPfeezCbzTh48CAqKipw5swZt69FTEwMqqqq7O+rqqoQExNjD2GpMXfuXDzwwAPo06dPq/vt378/hg4discee0z3OOJYao9JjYSEBFRXV7f4XH2NtdfNU6xYsQLTpk2zv79w4QLq6upw+eWX2++Bm2++GWLe24QJE+zihE8+cTplzo6YmBiMHj0aBoMBPXr0wJIlS/Dtt9/ar4PRaMSvfvUrjBkzBpGRkZg7dy62b98Os1nR9VRXV7v0cCUkghmSkJwgMzMTERERKC0tRWVlJSorK1FVVWUPKb3wwgtgjOHgwYOoqqrCxx9/3EKNpjXi9957L7Zt24aCggIwxvDcc88BANLT01FQoHT1LSwsRHp6ervP4bLLLsNLL72Exx9/3C2lHAAMHTrUISR04MABDB06VHfdzZs347333kNqaipSU1NhMpnw61//2qkSrKmpCadOnbIf5/Tp0w4k4+pYw4YNQ25ubovPTSaT/X/1dSssLHRQs2n/tATy448/ori4GFOmTLF/lpycDKPRiCNHjtjvAbPZbH8Y+Oqrr1BTU4Oamhrcd999uuN2BXF/iO9m2LBhDveMdjkAHDt2zCGkKiHRmSAJyQnS0tIwfvx4PPXUU6iqqkJzczNOnTqFLVu2AKAn1ZiYGHTr1g1FRUVYtGiRy/2dOHEC//nPf2CxWBAZGQmj0YjQ0FAAwD333IPXX38dFy5cQGlpKV577TXcf//9XjmPadOmoaSkBGvXrnVr/alTp+Kdd95BUVERiouL8fbbb7eQLwts3rwZhw8fxv79+7F//36kp6fj//7v//D4448DAJYvX273KI8ePYo33ngDv/zlLwEAAwYMwIgRIzBv3jw0NDTg888/x8GDB3HnnXfqHuvGG2/E3r17W4gW/vSnP+HMmTMoLy/HggUL8Jvf/AYA0KtXLztZ6P1pCeSjjz7CnXfe6eChhYSEYMaMGXjyySft51FUVIRvvvnG6fVrampCQ0MDbDYbbDYbGhoa7FL3nJwcnDhxAs3NzSgrK8Pvfvc7jBs3DvHx8QCABx98EJ9//jn279+PxsZGzJ8/H1dffbXdIyoqKkJ5eTmuuOIKp8eXkAhqOIvlBdOft2TfWlRWVvKZM2fyjIwMHhcXx0eMGMFXrVrFOef88OHDfNSoUTw6OpoPHz6c/+EPf+AZGRlO93vgwAE+ZswYHhMTwxMSEvjEiRN5UVER55zb8wWpqak8NTWVz5o1yy5v/u677xz229qY1bJvgYULF3JxjX744QceHR3t9Bo1NzfzZ555hickJPCEhAT+zDPPOOSPhgwZwj/++GPdbbXjmj59Ok9JSeFRUVG8d+/e/Omnn3aQbf/888987NixPDIykg8YMMDpOQlMmTKFf/rppw7HW7BgAR88eDCPj4/nU6dO5bW1tS73oYf6+noeHx/PN23apLvshRde4H369OGxsbF80KBB/I9//KPTfc2dO1c7pYHPnTuXc045yKysLB4VFcVTU1P5Aw88YM+pCfz5z3/m6enpvFu3bnzSpEm8sLDQvuytt97iTz75pMfn5woyhyThb8BFDolxN0M5gYzRo0fz3bt3t7rel19+KYurBjGOHj2KadOmYefOnWCMISsrC8uXL+8S1aotFguGDx+OH374ASkpKcqCCxeAkhIgJQVoQwkt+ZuQ8DcYY3s457olVwKx2reEhC6GDBmCXbt2dfQwOgQRERE4fvy444cXLgCffgrYbEBoKHD33W0iJQmJQIHMIUlIBCtKSoiMUlLoVda5kwhySA9JImiRn5/f0UPoWKSkkGdUUkKv6lCehEQQQhKShESwont3CtO1I4ckIRFIkCE7iU6PF154AYsXL+7oYfgG3bsDQ4c6JaO1a9fi7rvv9vOgJCTahi5PSNu2bcOVV16J+Ph4JCYm4qqrrupyiXNtW4SsrCwsXLjQ43140koCAKxWKwYNGoSePXvqLt+yZQsYY3jppZd0lz/44INgjNkLturhwoULWLFihb2GnrewaNEiXHrppYiNjUWfPn1anYfmClarFVOmTEFWVhYYY/j+++9brOOsLYXeBGDGGN5++20AwG233YbDhw/j4MGDbR6fhIS/0KUJqaqqCpMmTcKsWbNQXl6OoqIizJ07FxEREX4dh7pHUEeisrISNTU1WL16NebPn4+NGze6ve0999yDkSNHoqysDL///e8xZcoUtNZaftGiRY4SZhUaGxvxP//zP8jOztZdvm3bNnvVB1f48MMPccstt8BoNLZ+Eh6Ac44VK1agoqICX3/9NZYsWYJPP/20zfu7+uqr8fHHHyM1NbXFstLSUtx888149NFHUVZWhry8PIwfPx5AywnAhw4dQkhIiMME43vuuQdLly5t89gkJPyFLk1IohTNPffcg9DQUBiNRowfPx7Dhg0DQMbsqquuwqxZsxAfH49BgwbZW0YA3m9PkZWVhUWLFtk/e+ihh3D+/HlMmDABsbGxuOGGGxyqfN91111ITU1FfHw8rr32WodK2dOnT8fjjz+OiRMnIjY2FtnZ2W4ZcAAYPXo0hg4d6nbbCk9bSQDAzz//jI8//hgvvPCC7vK3334b48ePx6BBg1osa2pqwqxZs7BkyZJWx/bVV19h7Nix9veinceCBQuQnJyMrKwst+rQafHss89i1KhRMBgMGDhwIG6//Xb8+OOP9uXHjx/HjTfeiMTERAwcOBD//Oc/ne4rPDwcs2fPxtVXX22v3qFGa20p1FixYgWuvfZaeyV5gJo2rl+/3uNzlJDwN7o0IQ0YMAChoaGYNm0avvrqK92WDjk5Oejbty9KS0sxb948TJ48GeXl5QC8255CFE9ds2YNNm7ciNzcXHz55ZeYMGECFixYgNLSUjQ3N+O9996zbz9hwgScPHkSJSUlGDVqVItyOKtWrcLcuXNRUVGBfv364cUXX3TruuzYsQOHDx92aFvx2GOP2QujauFpKwkAmDVrFhYsWKDruRQUFOBvf/sbXnnlFd1t3333XVx77bX2BwdX0Gtbce7cOZSWlqKoqAgfffQRHnnkEZw4cQIAsHDhQpdtK/TAOcfWrVvtdfhqa2tx44034t5770VJSQlWrVqFxx57zOX1cAV32lIIaAvEAtSyIj8/v0UxWwmJgIOzEg7B9Nee0kFHjx7l06ZN4xkZGTw0NJTfeuut/Ny5c5xz6h6alpbmUDpnzJgxfMWKFV5vTyE+U5flmTx5Mp85c6b9/Xvvvcdvv/123f1XVFRwALyyspJzzvm0adP4Qw89ZF++fv16PnDgQN1tRSfT+Ph4HhkZyQHwp556yu2uqZ62kvjXv/7Fb7rpJs65fmmk2267zV4iaNq0afzFF1+0LyssLOSXXHKJ/TwB8JMnTzodm8FgcGin8d133/HQ0FBeU1Nj/+yuu+7ir732mhtnqo9XXnmFDxs2zN7J9dNPP+VXX321wzqPPPIIf/XVV1vdV0ZGBv/uu+8cPnOnLQXnSlmo6upqh8+tVisHoNuGRJYOkvA34KJ0UJeXfQ8ePNjeEfX48eO4//77MXv2bKxatQoAdZFVV2AWLQ7U7SkEmpub7e0QSkpK8Lvf/Q5bt25FdXU1mpubkZCQ4HBsdesEgR49etj/NxqNLd6LStM2mw0vvvgiPvvsM1y4cAEhIeTslpaW2ot1qvMRrbWsENsyxrB48WKsWrUKjY2NLbrA6sGTVhK1tbV49tlnsWHDBt19ffnll6iurrYXSdVi9uzZeOWVV+zn2Br02lYkJCQgOjra/r49bSuWLFmCFStWYOvWrfbcY0FBAXJychw8qqamJjzwwAMoLCzEkCFD7J+39p0Ajm0pAGr7kZycDLPZ7HAdRIHYmJgYh+3F+cu2FRKBji4dstNi0KBBmD59ur2FOUAVlrmq3p9oceCL9hSeYOXKlfjiiy+wadMmmM1m+yRR7TE8RWhoKJ566ilERkbiz3/+s1vbeNJK4uTJk8jPz8c111yD1NRUTJ48GWfPnkVqairy8/OxefNm7N69297S4h//+AcWL16M22+/HQBVGH/mmWfsywHgF7/4hW4HWEC/bUVFRQVqa2vt79VtKxYsWOCybYUaf/vb37Bw4UJs3rzZQSmYmZmJsWPH2u8LIRb5y1/+0kKE4A7caUtRX1+Pzz77rEW4DqCWFVlZWYiLi3PreBISHYUuTUjHjx/H22+/bW9gZzKZsGrVKofy/iUlJXjvvffQ2NiIzz77DMeOHcMtt9zi9fYUnqK6uhoRERFISkpCXV0d5syZ49X9P//883jrrbdatHvQgyetJC699FKYTCZ7y4rly5ejR48e2L9/PzIzMzF//nx7y/D9+/fjtttuw4wZM/DBBx8AIAHFgQMH7MsB8qp+9atf6Y7tlltusX8nasydOxdWqxVbt27FunXrcNdddwEA5syZ47JthcAnn3yCOXPmYOPGjejbt6/DvidNmoTc3Fz8/e9/R2NjIxobG7Fr1y4cO3bM6TW0WCz2a221WtHQ0GAnnNbaUgDA559/jm7duuG6665rse8tW7ZgwoQJTo8tIREo6NKEFBsbi5ycHGRnZyM6OhpXXHEFLr30UvscDgDIzs7GyZMnkZycjBdffBGrV69GUlISAEogW61WDBkyBAkJCZgyZQrOnj0LgAze3r17ER8fj4kTJ2Ly5MleHfvUqVPRu3dvZGRkYMiQIV7vkTNx4kQkJCRg2bJlAICZM2di5syZTtf/9NNPsXv3biQkJOD555/H6tWr0f3iZM2tW7favQuDwWD3blJTU5GYmIiQkBCkpqYiNDQUsbGxDsuNRiOio6ORmJgIAEhJSXFYDiiN9PQwdepUbNiwAfX19fbPUlNTkZCQgPT0dNx3333461//qqvmc4WXXnoJZWVlGDNmjN17EtcnNjYW3377LT799FOkp6cjNTUVzz33HCwWi9P9DRw4EEajEUVFRbjpppvsXYsB4Prrr8eCBQswceJEpKSkIC8vr4VH+NFHH2Hq1Km6XveqVau8Pg9LQsIncJZcCqa/9ogaXOGDDz7gV111lUfbSAQeXnjhBf7uu+9yzvVFFJ0Za9eu5XfddZfL5RIS/gSkqEGiK2PBggUdPYQOw6233ur/fkcmE5CfD2RlATrCHQkJZ5CEJCEh4T2YTMC8eUBTE2AwAHPnSlKScBtdOofUGqZPn45t27Z19DAkvIhx48bZRSwSPkB+PmA2A4zRa1dvESLhEaSHJCERrGhn+3KfIDwcOHhQ8ZDcmMcmISEgCUlCIhgRqO3LrVZg2DAgJgaoqaH3EhJuQobsJCSCEYHavjwrC4iPBzinV1WR11ZhMgFbt9KrRJdEl/KQjEYjvvzyy44ehoRE+1FeDmzfDjQ3AyEhQEICcPq0x7vxdlsOZGaSkMFTlZ0UQ0igixHSDTfc0NFDkJDwHq6/PjDl1ZmZno8nP5/IKCuL/s/PD6xzkvALuhQhSUh0KrTF8AcqsrLIM8rPp1dPQn0SnQaSkCQkJDoebQ31SXQqSEKSkJAIDHQmj0+iTZAqOwkJCQmJgIAkJAkJCQmJgIAkJAkJCQmJgIAkJAkJCQmJgIAkJAkJCQmJgIAkJAkJCQmJgIAkJAkJCQmJgIAkJAkJCQmJgIAkJInAg6z6LCHRJSErNUgEFmTVZwmJLgvpIUl0PNQekbrqc1OTbIEtIdGF4HdCYow9wRjbzRizMMY+1Cz7JWPsOGOsjjH2HWOst7/HJ+FnCI/o/ffpNTxcVn2WkOii6IiQXTGA1wHcBMDeHYwxlgzgXwAeBvAlgPkA/gHgig4Yo4S/oO2DY7XKqs8SEl0Ufickzvm/AIAxNhpAT9WiyQCOcM4/u7j8VQCljLFBnPPj/h6nhJ+g1wdHVn2WkOiSCCRRw1AAB8QbznktY+zUxc8lIXVWyD44EhISFxFIhBQD4ILmMzOAWL2VGWOPAHgEAHr16uXbkUn4FoHqEQmRhSRKCQm/IJAIqQZAnOazOADVeitzzpcCWAoAo0eP5r4dmkSXg5SfS0j4HYEk+z4CYLh4wxiLBnDJxc8lJPwLKT+XkPA7OkL2bWCMRQIIBRDKGItkjBkAfA7gUsbYnReXvwLgoBQ0SHQI9MQWgQpfVLaQ1TIkOgAdEbJ7CcBc1fv7AczjnL/KGLsTwBIAHwPIAXB3B4xPQiJ4xBa+CC3KcKVEB6EjZN+vAnjVybJNAAb5czwSEk4RqGILNbTzuPLz2z9mX+xTQsINBFIOSUJCwlP4IrQYTOFKiU4FxnnwC9RGjx7Nd+/e3dHDkJDoGPhCni4l7xI+AmNsD+d8tN6yQJJ9S0hItAW+CC0GQ7hSotNBhuwkJCQkJAICkpAkJCQkJAICkpAkJCQkJAICkpAkJCQkJAICkpAkJCQkJAICkpAkJCQkJAICkpAkJNRwVsNN1naTkPA55DwkCQkBZzXcZG03CQm/QHpIEhICzlpOyFYUEhJ+gdseEmMsE9SvqBuASgAHOOcyfiHReeCshpus7SYh4Re4rGXHGAsD8OjFv74A8kAdXGMB9APwM4C/AljKObf6fLROIGvZSXgNzmq4ydpuEhJeQXtq2R0A8B8QIeVwzm2qnYYC+C8A9wHYB2Cod4YrIdGBcFbDTdZ2k5DwOVojpHGc8xK9BRfJ6ScAPzHGunt9ZBISEhISXQouRQ3OyEhnvQveGY6ERDvhS3m2lH5LSPgULj0kxtjfAbTaMIlzPtVrI5IIPARL/sSX8mwp/ZaQ8DlaC9nl+WUUEoGLYDLEvmy9Ldt6S0j4HC4JiXM+z18DkQhQBJMh9qU8uzNLv4PFA5bo9PCoUgNj7DoADwDIAFAE4GPO+X98MTCJAEEwGeLMTPLgfGFcvbHvQDT8/vSA23L+gXjNJHwGTybGPgxgAYDlAHIA9AKwkjH2Mud8mY/GJ9HR8LaR1xoYbxscX8qz27PvQA19ttcDdvf7a8v5B+o1k/AZPPGQngVwI+f8gPiAMfYPAGsASELqzPCWkdcamBkzgGXLuobBURv+I0eAL74Abr+94883KwuwWIAdO4CEBM88YE8Ioy3EF0zhYgmvwJNadkkAjmo+OwEg0XvDkejU0NaE27Wr69SIE6HPI0eAgweBLVvImAeChJxzgDF69QSe1PhrS+g3mMLFEl6BJx7SNgDvMMae45zXMcaiAbwBYLtvhibR6aA1MGPGAPv3dw2DI0KfX3xB74cODYyn/vx8IDISGDTI8/F4QhhtCf36MicoEZDwhJBmAvgUgJkxVg7yjLYDuMcXA5PohNAzMOnpXcfgZGZSmM7fJOwqz9MWL0S9P3cIQ72+CL8B7pNSZ78vJOxwWVxVdwOq+p0GoJhzfsYno/IQsrhqgKG9QoXOrqzy5/m5k+fxZDyeCg3U6zc0UGgwIsKznGFnvx+6GNpTXFUPZgC2iztOBwDOeXHbhyfRqdBeZVRXUFb586nfHWGAJ+PxVGigXj8nh/JUV1zhfniwK9wPEna4LWpgjN3AGDsNoALAGdVfAGRlJQIG7W1mF8jN8IKxll1bhQHOztXT/anXj48nJZ8nYwnk+0HC6/DEQ3ofwHxQHqneN8ORCHq0VxnlLWWVt8I8Yj/h4cEpUVfn7cLDXedvnJ3rjBmA1apcS3XeCCDicnad9db35HuRSrsuBbdzSIyx8wDS1T2RAgUyhxRg6OgckrfCPOr9lJYCRqOijnvoIeCaazzfpyfH9iSv446wwNU1cXauR44A9fVAcnLL7fT2CXg/3yNzSJ0K3sohvQvgWcbYQu6pEkKia6G9OZL2bu+tCZXq/dTWkmH2xpN6awZWbwKx2kNxta6WMMRxWrsmzs61vp7ISW877T5zcoCvv/a+F48GpkYAACAASURBVCmVdl0GnhDSGgDfAHiBMVaqXsA57+vVUUl0TvjrSddbYR5t/uPpp50Tg7twx3vTVnWYP1/fQ9GuqyYMPVJzdU2cnasI3+ltFx5O3lRtLW0D0PESEoDjx4mgJJFIeABPCGk1gK0APoPMIUl4Cn+qpbw1odIXEzNb81RMJuD8eZJICw9FVFIwm1uu74x8tcexWl2fi6tz1ZsrZjIRURmNNMann6b1Vq8G1q2jddasAbKzJSlJuA1PCKkPgJGc82ZfDUaiE8OVIfaF5+QszOPsWM4+b0u4qK0TUdWkzRjJo8vLgeXLyeMwGMgr0Z6nHpHoHae1c3G2XO9z8X2KnJrVSutMmQJUVgKDBwMVFR1fiUIiqOAJIX0B4HoAm3w0FonODGeG2N/tD/SO5c0xtLYvV56INlS3bh1gs5GHNHo0vVqtLY+pRxi+Lrvj7PvMzqY8UkWFVMVJeAxPCCkCwFrG2FYA59ULZAtziVbhzEB6S4DgDpwdy5MxtObN5edTaC0mRj/EBjj3RNRGXogJevYE8vKAqir63xMD70sxgLPvs7PUn5PKvg6BJ4R05OKfhETboDWQIl9isfhnnomzp3p3RRDueFLh4VTNW6yjDbG5gnbO0LJl5GlcfjmFwgItH+NJiC+YIKtDdBjcJiTZzlzCq1D/6DkHJk3yrcEVT7x6EurMTPp81y6qQO5sDHoyZ+1TtNUKDBtGHlJNjX6ITTsm7VjE/12p8GwgwZ9eu4QD2lLLTkKi/dD+6Hv08H3uyGymUNjLL7f01ERlgv37iQjEGJ0JBRoaSEUWEeH4FC3WOXvWdcM7d57Cfe1peDMs1ZlCXLI6RIdBEpJEx6A1tZm3pdZmM3DqFL3Onw/85S+eT/BUh9TOnyfRgd5TtDsN7zriKVx9XQH/CTmCDZ0lDxaEkIQk0TFw9qP3hXHLyiLPyGymCZxGoyMBaMkRcE4WgphMJiItLaHm5NBxBg1yLXv2xlN4e9pG3Hyz9wixM4a4gj0PFqRolZAYYw8D2CBbTEh4Ha7mt3jTuGVmUphu/nwio/h4RwLQkiOgTzbafWoJ1WSiMF5eHv1dfrmjvF2IFUQOyxkhu0My7pYMcuYFAt4LS+mRa2cK4Un4De54SGMAvMwYqwCwHsAGANtlPTsJr0BruIRxE0U9PVGpuUJ2NoXpXFUqUH/mKmSjHrO6wGp+PuWUJk0Cjh0jZZx6ntPZs8ChQ8BllwFpaXQM9fY5OY6k6apYqVZeLgQWzip1h4c7koYQkLQm5HAHeoTemUJ4En5Dq4TEOX8UABhjlwG4BcAbAAYyxjaDyOlrznmpi11IBBN8/WTrTh5jxgzFMC9bRiIDb4zFnTCMM7JRL3dmbAWZVlTQnKHsbPo8P5/I6OBBqrxw8iQQFdWyWsX8+URk8fHAJZe4LlaqlpcDJJ1PTGxZqVtdB0+tMARaCjnaS0pi+61bO18IT8Iv8ET2fQjAIQBvMsbiAdwEYCKAtxhjhQDmcs6/ae+AGGPfA7gCwMVfGoo45wPbu18JN+Dr5LR6/xYLMGQIPd2L8jPCcFmtZETdNWje7H3kSeFTvfySXq+gigryjEpLqfJCZSVQVOTo/eXkkEzcaFTUgIDzSbZqefnPPwONjY6Vuo8coWMIpZ8o7yNI1pekIVVqEm1Em0QNnHMzgH9e/ANjbIw3BwXgCc75ci/vU6I1+Co5LQjj/HmlGvS6deQ1mEzAhQsUwhK5B3VxUTG5VN0EzldqMXUY7OxZ4IsvgNtvdzymNvSlly/RliMqLaXPzGYi4qoqIDJS8f4AKkpaVETr9+1LOS/A+STbrCzypJqa6NpxrozpyiuBzZvpOuflkTemzZtpK3W3RhqekL5UqUm0EV5R2XHOd3ljPxIdDF882aoNc0MDyaGPHaNl/fopZXI4B4qLlTASY5SLycxsmRNRv/eGWiwnh3IpRiMRgMVCXg1A4SztMdWhr+Ji/bzPF18o3l9tLYXqEhJo301N5AGqW3LbbMDAgcCZM8Bdd1G4b/VqIpuUFLo+JpPrzq05ObTN1q1AYSFdv6goYOxYhVjFd6Kt1O3qmunltvTW1yPmQIUUXQQkAlX2/QZjbCGAEwBe5Jx/r12BMfYIgEcAoFevXv4dXWeFL55stV7XpElAWRnwz3/S03tYGD3RV1QQKWgny1qtjp9p1wHaRqLCIFVUAM88Q/u0WoEBA0iYcPQoEUFTU8tjitCXq7yP2UzkBtCyBx4A3nyT8kf19bTP7GyF1PbuJU8RADZsAEaOJMVeURH9DR6sPxFXK3SIjCQSy8ujcfXs6UhG6u9EXanb1XXSnqO47lqFoa+FDP4MzfpzPBJ2BCIhPQfgKAArgLsBfMkYG8E5P6VeiXO+FMBSgFqY+32UnRXefrLVel2ZmWSwExPJaxBtCgwGUnvt39+SXNSqu169HNfJzqY/TwyDyUQkZDYDJSXkufXpQ7mYykogNhYIDaWcTny883Hl55PXYDRSmDEhgT4Xxh5QvBOAyCIsDMjIINIQKrz8fCIOxiiU1thIhKVW7I0ZQxN7tZ6g2iiqRRXO6t+ZTHQtTSbX4Tp1mJVzuh4lJXQOFRXARx85GnRfz0XyJuF5Y6ydbTJwgCDgCIlznqN6+xFj7B6Quu//ddCQJNoDrdelfTqfNIk8IUEmevXbbrsNWLiQDP7atc5beufnk7fhqt13fj4Z5D17gOho8taammhZaCiQlESvQ4cCv/mNYtD1xiUIoKaGyCQ6mpapO6+OGaMY9sREOoeKCjqGUOFlZdH+i4uJjNQkKBR7w4eT96UmETWxxscDixa5ntsUHg68+y6de2MjLdML16mNbXk59WISYcwBA4ClSxUlnzDovhYyeJPwvDFWXxNwF4VLQmKMvebOTjjnr3hnOPq7B8B8uP+uB3+HGrRel958GL11TSYywn/5iyJ0iIpyVIuJddasIQNx8CDlphgjYYAw+up6dqdP076io+mJ/7//m7yisDDgp5/06+tpxyWu39ixtL8+fUgI8e23jnN/fv97Gnt8PB1LTV5q3Hkn7Ss5uSUJirlFIufzwAMtifX4ceDDD4Hp0x3l6lpxRU2NcuywMBqn9n5QG9vz5+mzqCgKY/buTfuor3c06L4WMniT8LwxVqkk9Ala85D8SvmMsW4AsgFsAcm+fwPgWgCz/TmOTgu18dbmIvwFd42BMKRnzpDBj45W5NDixy/WKSqinMnIkfQkf/IkiQTUNeuEgu7UKXrqt9mIJAYPBu6/Xwl/ffcdXaP4eEXdp66uADhK12trKZR15AiFtk6cAHbvBpYsAdavB77/nq5zUxNw9920H3U3VfX+tN+HIEEh0RZzi5YuJeIymcjTsVrJ09u8GSgocNyHmlxqa+m4paV0rMGD6dy0x1cb2/h4oFs3Wh4RoYQxn35av2q6r+4lbxNee8cqlYQ+gUtC4pw/6K+BXEQYgNcBDAJgA3AcwB2c8xN+Hkfng9Z4T5rknxbTrbVYcAZhSAcPJhJJTqYw1623tlxn0CA6p8JCIqyGBiAuTpFCiwoQop5dYiLlcCZMUPI7Yr6QyK3YbOTdVFXRfi+/nAzxzTcr0vDiYvIcxowBNm4kcrBYyGvJyVFECuHhygRWgwHIzSUvKTeXHg7OnlVCe3rfh5ogROM+QTAAkURSEhGy9jtVy7tDQ+k6hoXR/7NntxSN5OSQZ6idRCs8NWfhUH8g0JR7gTaeTgCPc0iMsVgAyVCF0Tjnp70xGM75BVCpIglvQ2u8hQLLl6GG9iR+1Qn60aMpnLVlC4XUdu1SnuQbGihclZUFNDeTCmz/fiKU48cd5+5cdRUZ/9RUIiRBRiKUt20bUF1NnkBxMRASQsRTUkJGHSBPRF0hYehQIr7ISDLWNhuRDQCMHw988AGRVLduwLlzQF0dbZ+URIQQH0/HBJS5WFqon8ZF+E54Lw88ABw4APzwA52vWqSglXdPmgTs2EHEJdR1arJz1lJDjEEiMNCJ1X1uExJjbAiATwAMh5LXEeq2UO8PTcKrcEeB5Qxt/QGoJ5o6a+ftDHpiiB07SBRw/Dg9yWdnK20empvJ82GMDHR4OJGDyUS5mGefpVBaUxORw4wZjuGwmBjaR0gIvQL0P2NEKEL2PHKkMjeotpY8m8GDyXOaM4e8JMZo39nZwKpVRKAGA4XUDh2ia1FTQ8ft1YuW9e8PPPVUS+WcnlepzS2JvFhmJp2b+vqrBSSihJA273Hzzco2zlpqCHRiYxgU6OTqPk88pD8D+A7AdQB+BpAFqmu33fvDkvA62hrzbs8PQNRbs1jIyIvJps6O48oIFxfTOlu20DjWrKHPIyKAK66g3Ep5OXkwISFEHkJJlp9PnhHn5C2I0BOgELXZTF4O57T9ZZeRx2AyEQmOHk1jWLWKPJr8fDqn2lpK9g8bpsijGxrISxOS9Oxs2s+6deQhhYQoeaWyMhrrPfe4P59Hm1uKiaHP+/RpGabUCkjUEnnA8VgzZrhO1HdyYxgU6OTqPk8IaTiAGznnjYwxxjk3M8aeAXAYwMe+GZ6EV9GWmLezH4A7T8pWKyneTp6k/5cuJQ/DlcxYz9CJSgF1dUQyv/ylY05GkFFICHkICQnkwaSmkgEWZCZyOv36KcZWEHVODoXUzp0jgrj7bmDBAiKXujpg507KJzFGXo3VqtSky8+nia3l5URSjJHncsstjuT68svkReXmEnH06QP813+Rd6TuUuuuwVGTqZCfq0N22lCf+L6c1bOzWl0/tHRyYxgU6OTqPk8IqQEkOmgEUMoY6wWgAkCSLwYmESDQ+wG4+6SclUXG2WajcJq2MZ6AK0MnKgUcOqSE0M6cUSah3nYbEZ3NppTLiY4mIy+8jvx8yp316EEk8otfEAGJsF9mJiniDhyg8zl/nkQKAG13+DCRUng4eXsXLijCh8JCIoGoKFpeX09jPH2a9jlzpnKe2dkkzc65ONUuM5OIa8cOCidqFW56dfzU0BKOkJoL9Z76AUTv+9L7brUPLXoTb7XGUIbx/IdOru7zhJC2Avg1gA8BrAbwFQALgP94f1gSAQO9H4C7laIzM4FHHlEmtaondKr35+qpLz+fvImaGjK4sbEkcDhwgEJgot1C//7krezbR+G5qCileGl4OJFKSQnt649/pG0MBgrFvfUWkYzNRp/ZbKTSMxhorOHhRDKiAndICG3f1EQeW48eVBD13Dllnbo64E9/ov8nTnS8PmIisB4RX3NNSwGDmkjENVGTh9i3s5pzzgi/NeOm9+Ch15RQhvH8i06s7vOk/cSvVW/ngEJ1sQBWeHtQEgEG7Q9AEEhrTfRMJqqskJFB682YQZ/rGTBnhlF4WbGxdJz+/en/yEhF+lxfr6jxMjLI2+jZU5FAnz9PBNHURIRUUUHbR0cr6wwfruR/DAbytB56SCm6+tprlDuy2Wid2loaV0gIMHUqcO21wBNPUM6ssZGOlZcHvP461aa75RY6ny1bFBWbs5yNNkeklmTr9UdSTyAW6jtRc641whfHMplaemLOCFP9/eitIz5X70t6URJuwBOVXQSAZs55I+e8GcDHjLFwyCoKwQFvGoTMTPea6KlVdgB5OK6e1rWGESBDe+WVRDqJiS1ry6knaYaHA4sXKwVJRQvx8+eJbEJCiJCEcq62VukXBBCh1dXRuoWF5MkMH07bp6URIdXVKYKIkBCa09PQoPRw6t6dPLTmZkVx9+OPJMsODycSvOMOIkJtzgZwJAUtkQD6xv+ZZ0h5WFhIRKs9L71ckvgccO7luJOv0AsxavcF6O9fkpSEBp6E7DYCeBbADtVnowAsBDDOi2OS8DZ8EVZxp4meuqupkJxbrUQGekZO28CvtlZpVTF4cOu15f76V1q/Vy/aVhQ4zc4m1dyePXTMESOAyZMdy/QUF9P+6uooPFhaSiG+Pn1ozpDJRIQkxBSA4iF1707jCAujc+RckaKfO6eEAuvqaDu9OWCi9YaoRiHKHmkJ6+uvHa+dqAZRU0PnHBKi5JM2bKCxq+X9evdBW0N6QMt1cnLoYWDQIMdqFHpEKkN9Ehp4QkiXAcjRfLYTpL6TCGT4Qh3l6ulZXSladDUtKVFK3tTWUrfY9HTqGzRmjCJHFg38tm8nUhJ110SuRj3u8+dJci3CcP/v/1EuiHMKiyUl0fLLLiMj378/jWX+fDq2EDbs20fjEPmf+npSBnJOuapu3ejYjNH+GxsVefjQoXROAIXuSkpI0FBWRsQQEkLnJF579AAeflipsaeuM8c5hfnKyoD//V/g008dw2dZWY6eTk4O8Pe/k/rPaqVjhIbSOZ44oXiwl19OhVed3QfuhPRcQe3drllD55CXp3ioQMv9B5tiz1/eXBf3Gj0hJDOAHgDOqT7rAaDWqyOS8D58IRV19vRsMlEuRcink5KUMJnRSGTz44+0Xm4uvY+OBlasoP1YLMC//01hMKGqA8jrUdeWe/ddMsglJeSJRETQNrGx9BoRQfv96Sdg0ybyTnr0IPLYt4/k16dOKS3FIyPpNS7O8Tybm8n76NOHSM9mI6Mv2kjExpK44uuvKYx56aU05tpaIj2Lhf6EN/X735MyECCP7tAhIkyjkcJup08rk3vXrydBhF6tOVHnr7iYzqmoiMbKOXlyjJEXJmoAiu/JWc7KG8qt/Hy67mPH0nmNG6d8rlehvaPly+4af3/1T5ICEY8IaQ2AlYyx3wE4DeASAO/gYhtziQCGr6Siek/PGzZQgdLQUDLeL7wAXH21ohg7fpx+cOfOEXFUVChN8J54gozZ1q1EVNXV5J0kJZHXMmeO0kfp3DnyDBoalGNzrtRsE15ZUxMZ5aoqRQVnNlPvo/p6WleEHysqiLgYU6o1GAxEOpdcQq0XRM265mYir8TElvN41q8Hli+nYw8ZQgS0fTvto0cP6uqam0uEVFlJIbxx40iOfuoUjVfIy3NyiHi0BVnVdf7i4+nYGRmUR+rXj7YRHp9QN7aWS2ovsrLo+9izR7kX1CIObSmijpQve2L8veHNuXO8YPMafQBPCOlFAG+DwnQRoHlJfwMp7iQCHW2RirYlfFBSQq8REWTcy8ootAbQU7LJRGqwgwfJqNfVkfE1GmkdUfxTFP1MTKTJtJ99ppQHKisj42yxKMe1WsnwJifTsWfPJuJYvpy8DoOByKBHD/o/JobCZE1NtN/CQiInxpSKCwYDHa+igvI0oaF0LUJDiTT+53+ocoNYJtSGW7bQOVdXExFWVJDQYfNmIiHR/K+ujtqWnz9P4cT+/cnTstmIBIcPB1auJNI5fpwILTycPC+9On9NTURwERF0zX7xC/ps/HhHIgBaVmjQysvbYggzM6kkVWUlkeXx4zTGK67QN7AdKV/2xPh7I8LgzvF8EckIMngi+24A8Dhj7AlQcdVSzjm/qLST6GxoS/jAZKIKCaJdQUwM9Qf65BMywgMHUvWDBQso32I2k4GNiVGe5jMziRTMZoUsjh0jkoqOJq/BYiGDbbMp+ZmICDLUVisRwcqV9CoqaxuNREYxMWQwz59XckEAHae5mfYp1Hic07pWK61bXk5j6N4duPdepe1DXh4dY84c4Ne/JoNSVkbeQlUVXZeMDLqOlZV0fbp1I1I6cYI+37mTthkxgnJq48fT/iMjiXDWrSNSmzOHrp/Wu5g4kbwpi0UpmXTwIF2XxYup31JmJu3z/HklV3f8OH1Hnlbj0FtmMtGraEIYH0/XpTUD2xF5E0+Mvze8OXeO19FeYwDAE9n3Ys75bM45B3Dh4meRAP4N4GaXG0sEHzwNH6gJbORIetr/9lsiBFEB22IBnnwSeP55Iq6CAiIB0cIBIIOZmqq0zj51Cpg1iwx4dTUREmNk8ERYTSjaCgqUfE1NDRGIIJyoKArZiV5IDQ2KN6SGICaAyNJopH02NNC6FgvlbebNo8mweXlElmVlFKo6epTChmJ9sc9z54jIoqIUIuVcIeI9e4iwExMpfCfq34lq5iI8eeaMQkpqCLWcCCHm5NA1GjxYIbOzZ0lkYjDQGH/80fE81cbS1QOJ3gRcQFmfc5rHJYQbgZg30TP+roixvd6cu2TTiSe9ugNPQnbDGGPzOOdzAYAxFgVgHYBCn4xMomPhafhAPecoPJwM1blzZGiE19HQoBjz8nL64ZWVATfcoNRyCw8nErJYyKtqbqZ9XHcd5aYiIogAqqrIe7BYiNzKyykEJsikrMxxfHV1FJYTy0NDW5KRFiJsKIq1NjTQ9hYLkWNKCp1bSQm9NjfTuETIz2ZT9hUfT+v37EmeWlUVheeEJ2azKQIHQQg5OQphCQj5+Jw5ynVetIiWia66Yn4WY4psPiWF9hkTQ8cbMsSxWaC2lbyzahyilJOofi4m4AKO62u77bq6bzoqb6I2/v4gxi5ONu7AE0K6HcAmxpgZwP+BSgedAPCILwYm0cHwNHygnXPUp48iEGBMMZ6JifR/Y6NSfic3l4zBjBkkbsjMJIIROZGPPiLPSDx9JyYSScTFEfFUVSnhPWcQ3pB4DQlRSPbMGcd8lCBCm00hlbIyOqczZ5T5RHv20BhiYmgMgEJyajJqbibSKimhIqwREeQNWa3KJNvkZODBByn0tm8fXY/6ejrWDTfQ8WtrlWZ/p06R15iXRyIK0Vqjro6k6OPHK9L21auVuVCiAOv48eRRVlQolcD1qnFoH0jy8xXPSNvBty35D3cffHwZ1jOZSPZvNiutOrqgoCAQ4EkOqZoxNgHUXvy3AL7mnM/y2cgkOh6ePNFZrY5zjsrKKN8iZNKi5E9YmGLEi4qUHkO5uRTOS0ggL0qIChobyfgKMEb7SEykvIyYCHr2rGLc9SA8IkEYkZGkACwvV0JOAur1BBobSZkn8jMCNhupAAUhuYPGRiKDnj0Voh0wgMiouJiIubRUkYrv2UPXqKyMzl1UgBAQ+xCSeqtVaWUu+l4JZZ1aeq03uVjA2QNJVpbiGYkJvGJZW/If4jii4KwefOm9iH2bzfRABThWTHe2TRfO8/gSLgmJMfaazsc7AUwEUCGWc85f8cHYOge6ys0rDJXZTE/t9fUU1hI9iJKTFTIB6LOUFAoxHThAxoAxCvWkppLRjYggA62GULP16UPEN2gQ7efzz1uGydTN9jin9S5coM+tVuAPf3CURwv06EEEp4WWjADyWoqLHQnCHYhJtk1NdD7dutF98v33SthP5LNMJuDGGymkl5FB11co7ZKSSI1XUEC5JsBRIq4uy6RFaw8cestdec7aEJgn972o0ff11y0Jx5dhPbFvUdVj7FjqJOxs/3KukE/Rmofk7Ep/pVrWSiC+C6Mr3bzCUH3xBb0X1QJCQpT8UUYGeQXbtxMhjRtHNd5OnqRQks1Gr5dfTpLlwkLyFNThtKgo8rxKSiicd+6c0vCOsZahMgGbjTwMISgwGOi42jCfCC+GhjruQ0jRtTAYaF2jkcbjDIzRNbFaad2wMBpLt25KpYnwcBI+iAnBouJEYyMVqW1oIA8oMZE8TiGDX7tWkdSvWaOE4dRP+d6uZejNiaStEY4v5dDqfcfHuyYjd8Yq0S64JCTO+YP+GkinRDDevO4YLlfrpKYq82wiIsgrEXN2li+n+URCdRcaqpCRKH1js1HC/aGH6DiLF5NyTeSc0tIUMjIYlHYRPXtSUt9icfSMACWPpa5DJ46lRXg47auykv6vrlbWFcdSh/NErkl4f5zTcbQKPhGCjIxU1he5KINBmfh7773k9VRU0HURMm6TiebzNDZSU7+qKtpO5DysVkXVBjjmhPzxYKS+Jzy979WkYLHQ92gy+WcSraf7lnOFfIrWQnYpnPOS1nbCGOvBOT/vvWF1EgTLzSuMiV7/Hb35J3rVnHNy6Ok8IoKM7ciRFPYShVKPHCEPxWIhQxwRQeGy/v2JlMxmJUx18CCNY+5c4Nln6elfdIsNDSW5tTDIqakUNktLI+8iL4/2pZV1i9yL8HSEKk5NGozRPvfto3XMZsdzZ4w8GlEgVogsOCfvJzqayMxqJdHCpk1Ky4qoKCUUWVCgKA9FPqiggHJlJSUUjhs2jCqam81UNslkousnVHIJCfQqWoBUVJD4Q3wvgpzUCfuePSmsl5PjXIKtZ5hzckhsImoO6m3nSSt0LdR5pNWrlVJM2soOvnqY82Tfvq520cXRWsjuO8bYFgB/B5Bzse0EAIAxFgLgvwBMBXAtgEt9NspgRSBNdHNmbNTGxGQiQzlypGMOQr2uVo0kevQUFREZTJpERLNuHeVWzGYyuiJ8JwqBWixESj17EpGIFhE2Gx170yZaf9QoJYTV3EwiBLPZMWcTE0N5E5uN9lNTQ4ZQeESc0xiETBxw9NI4V9Rr9fXO1Xpi3pB6Ii/ndD41NeQNJidTuLFHDzrn5mYiqpAQqhhhtSoeG0DHFNLykBAaf0gIeYWisGxtLe33t78l4i0sVOZtiflAS5fSq1olBigJ+z17iFTCwujBQetBCTKIjHR8GMnJoUnMgmxWrGhJSlqPqLVW6HoQ86hEn6tAjihoq11YLDTxWKtUVKOr5JLbidYIaSRI1r0UQF/G2GkA1aDGfH0BnARJwGf7cpBBjUCYe+AqZCOMSUIClZ9pbHTsJaTdh1aNBND2gwYRIR07RsdISKD8T0MDGUGDQSmEKiCqX588SccVooHcXHpdvpzUZw0NjjXrRHkgs5me+MV8G+FxiPyPGs3NimckIOb/MKbUp2ttbpJ2LAKi19OJE1TDLSyMxhkSQjX6Nm6k/8vKaHyVlcq8JgGhQBwxgry0q64iMhLt2NPTle9x/37g5puVFiC7dtH3Jr4Xdehs6FASUFRW0vVsanKcVyQKtZ46RQ8U6oeRXbtofbHurl0tCUkvEqAVOAgVnSujHSwRBcDxd7NuHV1bUWBXW0S2K+WS24nWckhWAEsALGGMZYJaUHQDUAHgIOe8yPdDlGg3XMX0hRE4fpyM8YgR9LQ/6EgEsQAAIABJREFUZYq+0kmrRgLohyiqAgwYQIn2Q4fIkJrNSl5AbextNiKBTZtaTmIVEJUatLmesjL9HJB4r5cbAhxzSNrjONvGHQgCFoQgJs/W1NA4N26kJ/+RI4lompqU+n6iUnlsLHlYqank6YjQXFoa8NRT+pNVxbGPHKGHgX79yHN7+mnluxMGPi6OPCvREyo8nAzlhx8q24qaeRkZChmMGUPrm0w0nrAwx/wO4DoSYDJRA0FRcFW0jHem+guUiEJrEL8bMfF40CAi9fnz6SHBnX5TEi3gyTwkEwCTD8ci4Su4evIURmD9evJMhIej/cGo9xEaSoZTvb3ozSNUdoMHA7feStscPaqo6ASEF+Os/bmAHom4SyAiLNaa19MeGAxk7KOjWx5HhBa//57I+eefaX1tSNBiIUPfrx8JG0T3W/WTtsmkiDbE95idTX8ffkge0IABShNEwNHAnz8P/OMfRHQ1NYpgZMcOIvj8fMpbTZ3q6MVkZ1OY7ttvafLtTz+Rl6R9yncWCRAVPETu7uxZ1wY5ECIKQOshNm3eq6KCHgaMxpbEE0yeXwfDk0oNEoGM1upwuXryzMwkz2fUKMVgaSXO6h/gihXA++/TD3HRIiXXUFBAXk9UFL2KQqipqfQULhASQkoygI7lK3iDiNSFVvXQ3ExhtauvJg9EO6cpPJxCcwUF9JqU5FjUFSAj1r8/8MADymRVNdQhH3WdOEFUR4+2bNsuIAy8yaTM9RGhVlEA1WgkD23qVPKMtcjOpvvh1CnPn/KzsoiMzpyh91YreXSB7AG5G2IT11Y98XjZspbE44nn18VzTZKQOgPc+QG19uQpJrYKg6V9ihM/lNJSyiEZDEQ2H35IZWhWrCCDJZrjJSWRASwsVCajhoYqkmihqktJUZrYAb71ZtoCIfUWYa7qasflzc1EMAcO0DpaWCxkhOvrafvTp5V5U8L7s1ppPpXZTMIF0Tl22DD6Lm6+uWWoTv0aEUEkdexYy1CrgNYoAiRuEFUw+vTRV9AJOHvKd8eTmDqVxCwxMcDhw9SAcffuwM2leBpiU/+2nFW/cMfzk7kmSUidAu78gNwNQTjLAzz7LD1Rl5YqifnaWgr15eQQ6SQl0TiiokjAYLEoc2+amhTRAmOKB1ZeTv/7k4jUZOAM0dEkNRfjFL2StPOLQkLotaHBeekiIZoQUvPmZiJiMamXMcpBvf++oja0WsmAi3E2NNB1DgtzVMMJiXVFBSkWXZGK1ii+9ZZ7YgOxrV517GeeISIVRV719pGdrSgxw8JaVpJwhvZ6C23dvj0htvaEHIMh1+RjD04SUmdAaz8gZ09e2pvL2Y8pJ4eeaMPCyIiGhysk06sXEZPJROE3zhUVms1Ghl3M7wGU6t0Afa5VmrUH2gmxejAYaC5Raanr9SyWliIMcX5qiONVVdG5xsa23Hdzs0I8RiN5S5WVynUJCaHrmZBA16OsjI4jCqFmZipEKGTvo0ZRGGzXLufKrtYMhyfGU29/OTkkVoiOJo/O2fwmdbjXWSUJ7XFamxPny3bgHSGu0MsRBlquyQ8eXKuExBgbAOBGzvmfLr7/GoA6E/1bzvkJr45KwjNkZiqVsseMaXmTiMRyTAy9queouHtzNTUp3kx0NBGRUM4xRk+9zc30FFxZSQa8vl5JZgsvSFQ0EHkUb3lGYWGtk1FICJ2HO3krrQfVmohChOSGDCHyVkvDRdmghgalArqYpAvQdkOH0rVLS6OwzyOPEEGFhyvS68GDqWZfUxPlpIQwZP9+x+9v7Vpg4ULaPi2t/YbDG4ZIm29Re1nqMKI4Tmlpy3lVnsio2+tt+FNc4SpHGEjwgwfnjof0PKjCt8CVAJ68+P+Ii8sf9OqoJDyDyaQ8Te7fTwZNfaNoW0OIGebu3FyiC2h6Oj29paUBvXsDEyYQ+ZlMpM5bvlzp1iqe+hsbiQDVoSzR+sHb0Ct8qoUgLL3QmzcgyhP17UvzkQSJRUeT2KCykr6HuDgipthYIjKLhTyHxERSyi1YoDToU8/9KikhErvhBpLV6xnsnBzg8ceV0keifXh7DIezeyU7m2TcFRVEfq7ChQLa+UlqYlHnympr6droeQvu3LvBpGzTno+6l5QzBHqX3TbCHUK6Fo4TX22c8/cBgDEWC2Cv10cl4Rla+4GqW0MIBZ07N5faYCQlUVgpMdFR9r16NcmZa2rIgDQ2Kh6ImIyqhmg3ri6Y6m+4yllpK4brQXh52nMTBrRfP7o+paWknuvenbY5e5auiShJJAQRYp8REYrcWz1HaORIWmfYMFKoNTWRCIGxlt/frl10jcV3XVHRfsMh7hVRpkhI9TMzKQ/VVsOovW8Bx0KnTz/dMhSpHo+rezcY5zS5a+gDqcuul+EOIaVwztXNXqaKfy72SOrh9VFJeIbWbmg9BZ07N1dODoWRBg2iJ9YxY8i4btlCs9NNJlKNidxGaCgZwtpaMth6oTHRVbUj4Ypwmptb954Ya+nlhYbSX3IyeYyi1p2YFzR9OuWGqqvpGEYjcM01lOxvbKR1qqro+wkPJy9n5066hvn5VI5o+nQ6lvjOiotbhmnHjKEHB4A8sEmTWr8e6ryNHgGIkLAoU7RsmeKFtye0pb1vxbwqvZBecbHj2DpTO3BPDX1Hih98fE3dIaRqxlgW5zwfADjnX4oFjLG+AHw4kaQLwxOX3J15RnrLxat4OlUbgPBwCiMdO0Y5kZAQpTZdSgpw7bUUlqqooPCTqHINKHN3ghGi/biAHnkxRmG4hgZFSCEIqrCQrkNEhNILql8/8pSSkojAIyIobJeURAKL6mpFVffkk/Qd7N2rVEA3GMg7Et/fNde0DNMCyoTa//1fKseUm6t04xVFcLX3gDYsKKTm2qduq1UpUySMoN7+XEFPRKOVomvJyNXYgoFs3IUn5xNM4UgP4Q4hrQcwH8ADOsvmXVwu4U20xSVv7YbWLtcrqDljhmLkSkuJaIRHwzmtzxgZO1EOCFD6+oiW36mpLRvcuaOACxS0FrJrblZ6H4lzCg1VJpyWlioJ6spKCrtlZlJuaM4c5RpdeSWwapVSmby+nsilpES5XvX1dP2XLSOSEgZZ/ZR85IjivQjDra0aIIrgau8psR8hMRev2qdurREMD/fsHnV2T4s/veXuji3Q4Ov8TjCFIz2EO4T0CoDtjLF9AD4HcA5AGoA7ACQAuMJ3wwsiePMm9LVL7qygplBzZWXRvKLDhylkJHr+qL2e48dJ4JCWRoZS5I44VwhMjWAhI3cgqoM3NCjnKSa3hoUp1bsZI3JITFS8l3HjKOw5cSJ9r4MG0bU0GGidd94hz6muTmn8d9lltK7aIKsJQpCP2nADjqIAQP+eEvsxm+lVSM2zslre02oj6Ok92tr66vCwmKPkamyBCn/ldzqbh3gRrRIS5/wcY2w0gP8FMAFAMoAyABsAvMM5d1IZswvB2zehr11y8ePv2dOxoOaYMfSEnp9PxvTSS+npvqioJcFUVSlKLjG3RqwTrOE6T6AuESRyTqKfE0Cqs5oaur4xMVQzbtEiIqyYGMrBWK30HZSXU95N1Jj7+WfH8OfRo7Tdzz+TZyUIQkj9e/UiqbfWcKtFAQB5SOp7ShCOmMdUUeHY2kKvvcI11yjn7ck96uqeFp1u8/LoT5Q/UpOgs/xWoCEYJrcGMNyaGMs5Lwfw0sU/BzDGhnDOj3p7YEEFb9+E7rrkbfHKtD/+wYOptEtmJv3gb7uNjJLRSHNe0tMVYylUZcIAC/m0aN3QlRAdrSgFRTkk8T9Axp1zunY5OcB//qMo1Gw2mieUkUHLH3mEiOubb5RJs4DSSj06mqTAkZHKcbQ5JEEqrgy3Nl+jbaq3dq1jawu99grqXJTehFwt1Peos3s6P995+aNg8wQ6cX7HH2hTpQbGWBKAewBMB81F6toVH3xxE7b2Q2yrV6b+8e/dqyi0li1Tksf9+iktCQwGIqULFyj5LvoWiTwLY2QE3ZkHFCxwZ46SaLSnt75439xMnk3JxabLnBOJiRDbqVN0zbdvJ1Lato2+z9BQpb1GWBiRUbduJCYRqjugZVM8tfeiB/U9JVpZJCSQh/ztt/oS7L17abwZGbTcWS5KD3r3qN4Yxe/HnfJHgY5OnN/xB9wmEsaYAcAkEAndBOAnAB8AeM8nIwsmdMRN2FavTPz4z5yhnMfBg2RkRA7CYlHUdA0NtN+UFDKykZH6oaqMDFKPdeTcIm/B1TwkNfGoVYUCoussQPX8amocxR2iwOzAgYooJCGBrn1CAoX0Fi6kag8Wi1KBPT2dasaJXFN4OH3WnoegrCz6fteto/dhYeSJqSXYmZkkwjAYSOp/+eW0rrv3nSf36M0302sgVijwFMHm1QUQ3CkdNBrANJBHlA9gJYDHOOfFjLEISEIi+PsmbKtXJsjziy8ocR4TQ6/19UREZ8/Sn80G/PijUlQ0IkJpA65O5nNOnpO27UIwgjHyRGpqHMlVLe0WYcrISDpv4SUBJFaoq6Nrp1doNSKCqixcuEB/zc1Ku/WsLCKo+Hil2duwYYr4QTux2dlDkLth3MxMCo1VVirFTidNIm9MbCsIZNgwJZQmCqW6c9+5c49qvahg9o4k2g13PKSdIAHDLzjnJ308Hgl30R6vLDOTEuGHDimVEx5+GPjDH8jA2mw0T0a0/C4uJuIRrRgERNXs0tLOkUMS5KqF6Ora2EjeSX29ch1ELi0igjzFHj3o/fbtSldcgZQUCsvV1dG1S0kh7ygjg66xyO0dPkzb1tUp+aHQUBI1NDaSyEE9lwcgw75hA/DPf5KqT28ukRaCXESxU613ohdK8+S+c2ddKQKQUMEdQnoNwP0ANjDGVgJYKYup+gnutIzQzi1yx1CYTMDSpURG4eGUKyoqIuPY1ERPzeLJXRCNCFGpcyXC2PqyyZ6/oRVuCAgxQmIihTojIui98JYuuYQ8jK1biYyE6lBMbI2KojComNtlMJAnGhJCRLRvH+1n4EDaPjyclkdF0fdVX08hu9pa4I03qAutaAUu2oNs3Urqx169yOtpzbi3Z0K1u6TR2rrBLgLo4g31vA13ZN+vAniVMXYtqGzQTsbYKVDo7t++HV4XhqeiBVfra380+flEPomJSpvt668nVZ2Y/xIbS/mLnBxHb6CrQHuuNhtdl/PnFe9FoKGBvJpf/pKuZ3g4/dXU0GtMDIXzLlxQ2koAlLdJSiJvpq6OJhyLcF/37rSv8nIKl4lq7Var4rUIwsnPp/dxcURc5eX06o5x93RCdVvQnm7GgQzZUM/rcLvsMuf8B875wwBSAbwN4EYAx7w9IMZYImPsc8ZYLWOsgDF2r7ePERRQhzLEZEh31zebKUdkMik/mvffp1eTSaltd8kl9CT98stUwLNnTyXBDpCRjY4mo2owKO0SuiIYo2vmLDTZ3Ax89hmRU2Mj5aBCQyks1707yelra2m5wUDXOj6ewp2iUd9llwFXXUUletLS6HuKjibhiclE2zY1Keo4QThZWfS+sZFISXyngWAc9e4/LTIzSX0XCOMVMJnI49Qbr4Cnv1GJVuGxXJtzXg/gEwCfMMbSQao7b+JPAKwAeoAk5esZYwc450e8fJzAhqehDLH+kSNkwADH+STqGH1WVktV09atlNAXLSMAIqLu3ckoms3kNYlEfGsdV4MJIjznqrxRfb3rcxa5tqgoqsQdF0c5utJS+lu7lq6fmLeVkECh0p07abJrUhJd54gIKqR6552033XrlO9+2DBFBTh+vGMIzZPur/5EMOaI3PV8gj3cGIBo8/whxthloBDefQAWeGMwjLFoAHcCuJRzXgNgG2NsLaiO3vPeOEZQYfRoCvOojY8zqNVzgNInB3Bdg0yomoTKq7ZWCVc1NFBuKTKSDGFUlJIL6UwQwoSEBAp96ZGSyKGJZaIFQ0SEMq/IYqHl5845dtA1GkksYbEok1sLCmgCckMDfc9NTS1VbmIS844dNLbx45XJsOqK24DyGmhP6cFotN0l0WAONwYoPCIkxlh3APeCvKJLAHwF8mS8hQGgfku5qs8OABjrxWMEPkwmmneyZw+9P3pUSWC7QmYmcPvtSvkfvZL+4seWkEC5CXXbaYvFMZkv5s2Iv4YGMradMZfEObWH0IOYqGowKPL2kBClYndSEglBTp9WutIyptSna2ggQqqsVPY1ahSF7XbuJIl3z55KtQztuBoaaHLtvn3ODWWg5jOC0Wh7QqKucmxS8OAx3JmHFAbgNhAJXQdgM4C3AHwBoBnAFC+OJwaA9vHbDCBWZ1yPAHgEAHr16uXFIQQA8vPJC4mOpvfqBHZr0BoAsT/1j8JiUVphr1ihEFb37o5hq5AQyhsJlVhtLT3t19Z2Dpm3O4iMpC6ucXFKm4eQECKOpibgppvo+/nFL0hybbOR93T4sELw4eGUFzKbaX82Gy2LiqKOrqJUjvB+1NWubTZSzpnNlKPq3l3fUAZyaCzYJop6g0QD9QEhwOGOh3QOJF74EMBUznmFWHBxYqw3UQMgTvNZHIAWk0M450sBLAWA0aNHd65HdiE6yMuj9+oEtjsQBsDZj2LsWMoZGY1UGmb9epqAyRjlOUTivaFBmX8zYgR5B4cPOyrMOisYIxIaOJBaks+eTYn5v/+dSLqxkT4Xc3huuYX+hBHbt4+ufXExEXhZmTKpNjSUSOjqqxWDJ0r5aHN95eVKRYfERNpOHdYTCMbQWCCjvSQayA8IAQx3COkQgMsAZAM4xRj7nnOfxWxyARgYY/1Vk3CHA+hagobMTCoj05YktTpM4OxHkZxMRrWsjJZ/9hmp7ESILi5OaaMg2pCLStOJiWSExRycjg7fudNy3BVEM0FBMuJ8+vShigljxii14l5+mUKW588TKbz4YsvioupXq5VILCODHi5qa+l4qanA/fe7RyihoUo3Wb3JqwLBGBrrzJAPCG2CO/OQxjHGeoMEDEsBRDHG/gGah3TIm4PhnNcyxv4F4DXG2MMgld3tAK705nGCAm15QtN6RDNmkJeTk+PYRyYzk4ilvp4S44mJVFxThO1KS5W5LCJncuYMGeyTqmIdHU1G3hiDwUCeYt++FEaLiKBrdv/9lBM6ckSp0p2ZCfzpT45GXzQ6zMlpSRaiEkJTEynqXFXH1iOU1avpQaB7dyKksWPdmzMkJMuSmDoO8gGhTXC3/UQBqGvsfMbY1SBy2gTqi+RtPAbgbwBKLu7/t11O8t0WmEyksDObFYWdyaR0Iq2qotIyI0ZQriI+npanpND/KSlEOPX1pA5Te0CMkad04ULL43Z0J1i9Y7vruYlSP/HxtI3wBA0Gkmzfcw9VtDAaHVVtaiHBs89Si3eAio8uWuToJXlilJw9hIieU8nJrZ+TzF0EDoItdxYAaMs8pG0gOfbvAPwKRE5ew8XeS3d4c58BC2+pcIQREu0jAKWdtqg1V1hIba4HDqTEemWlIuEeM4YmYjY2UjhK3WhOwGBwLCQqwDl5FVZrYHhMzqAN7UVE0MTge+8FvvySxn7sGJ13dDQR0xdfEBEIgs/JaVnxoqJCEZ+YzS1zBe1RYWVnE8mZzfR9ilyTq/vFk9yFVIFJBBjaPA+Jc94AYNXFPwk13Pmhe/NJVhihoUPp/dixJP8GgA8/pPBbWBjlhmw2MqKiFE15ObBqFRFKdDSRlcgdcU6GPDKSxhYRQWGs8nLl2MKDEiq8joCYE2SzkccWG0uEK6pzC28PUDw6g4GMfFISeR6MKfmkM2fo+ykqUqpThIbSnKCmJvIiX35ZqZBw4gSRea9eLXMFeveCCPOtWUPX1Nn3L3KJ+fl0jloVXnsma0pPSiIA4XbpIAk34U6pFMC7ZUfURig+nshIPJm//DKVkomLI6OZlgY8/zx9lpxMBri0lNRg5eW0jiAkMfcGIAFEXBxVEBAVBwRsNiIkPQ/KHQj1mbvbi/UYIxKNi6PzCgujsRmN5FkMG0YkwZhCLGFh9CfK8mRm0jmWlBBJ19YSOScmEln16kUEP2UKXZdTp8iTmj+f9jd7NoXy9MJpevfC/2/v3KPsqKo0/u10upPQMS8S8uykeQTCw2CgYwNCghAkEBJxCKI4S1GMg0tkQBFGXCxFHjIwM6jgQlEHFUZEw6DyGFRGEsJj2oTFgAOBkEgnTRI66dhJuvPoV/b8setY1dVV99a9fR+V7u+3Vq/OvbfuqXOqOuervc8+ewffW7PGBC3T/XdpdVyYeba/F+cmvPzyzCLDtDckhQzuSq/FIKnLpJBROJnWKurrzUp64gk/60N9vUXVNTQA995rk6zbczRjhrntKir86rAHDtjGTbeHprraz9nmctyddpot4Ls6QknXcYLHBYUm03ddmp+JE83t9s47tu41ZowJUXu7rf8ceqhtPL3nHutXMIrQRa51dtq1+8lPLAR+1y4bY0uLjWvWrN7WpnOfjRhh17u52cSsurrvRuOovwXA3ps1yyLv1q41UWpuNrGKE5BCbdbMpz1CSgQFqdAk/Y/e3yicsCsonDom3N6aNX1TztTUmCvoiivMDdXVZfuRXHTZIYeY8Iwfb0EP69fb6/Z2+6ytzU/G+tWvAosXm0W2c6eN3aXOyYSrqeTcZS6AIhsHDphoHH44cOGFwIsv2jrZn/9s41i+3IIWWlps3WzbNmu7o8N+tm61sbjouWOPtfM7t19lJXDOOb2TlN54o1lGI0b4UYvNzfF9dH8LwUg9V+m1tdXE84MfBJ55xnLWPfVUvFVT6KitQm3+5BoUKSAUpEKTy3/0fKNwovz/QPSaQFT0XdBq6+z0yyW0tJjojBxp+dWqq83CmDjRJu9p02zNw03IFRV+MbiXX/ZdaGPG2Hmdmyyqeqqjq8vacSIUdttl2mfU1WVBHM5ldsghZi25wnYzZlhfq6t9l2BlpY21uhqYOdPvW02NhVfv3OknQH0tFNxZX2+itHq1BYK4a1hXZwIzdmzviqc1NRbq7UTshz+0+xL8+2hsNDGtrbXz/eY3vsu12PQnCoxrUKQIUJCKQbHDPaPy0QE2Gbty1M5Sioq+C1ttlZV+pNjZZ/tJPN2i/kknAevWmfXR2WnC5Vxb1dX22S23mAC48uaVlXZsJjEKnn/YMBOftjYTJ1cqPdu6UkWFrRWdeKKNtbXVr3T71FNmhYwbZ5NmRYVF1T36aG8rp6nJxjt9ui9oQ4eaxRV0wbnjurtN8JylefXVfUXK0dlpYhl024VLLURlaQ9P8GkTAGYiIEWAgnQwUltrbqfHH7fXP/uZ/d6wwX7q6npnapg2zdaPZs8GLrus7xrTccdZBoITT/Q/nzLFjwRrbTVxmDrV1l1aW82Ft327WRSATfBuI21Xl7122SCcG2zIEJv0N23qXcrBVaIdO9aP8nNC5kpiuHarqvx1rYoKv/jdQw+ZW66tzURswgQbw8SJwCuvWK2h7m7LS3fvvX03oL71lh3rsioMGWJBHsGNwHFrQlEiFbxXmVy4zqIOZ2l3E7xzizU3p0sAuAZFigAF6WCkpsai3XbutMXxtWttEp4/36yY4I7+/ft94Xr99ej2XPXYESMs2s5N1kuXmmA1NFgOtw0bTIRc6PiYMfbvfftM0Hbt8vcrdXX5e5MqK63tK680wbn/fptonSgNHWrvjxwJnHACsHKlH6pdXe2v+wDWrgthHzLE1ofGj7d+Dxtmfdqxw9av3n7bxrJ3r12j+nqb2GtrzUoBrB933GFiMmSIWU2jRvnW2dNPW9YGwL7b0WFtdnTY62xCkdSFO2mSCWxwgg9aRa4/aREAZiIgRYCCdLDi0tK4tYs9e2wiB4AVKyw4oabGRGXnzt6uvODk0dhoYjB5sq0D3XCDHwrtUt0Ado73vx949lkTwZ4eYMECq+sjYm67IUPs9b599rkLse7psdQ8GzaYWGzZ4gcyACYibW0maM895wuVC0kfMsSEyEXfqVp748fbmg5grrbmZj8xrLOiOjutvy6sPRw88MADFl3n9l0NG2aitGOHhZD39FiGi9WrfUvu1FMtEOHhh63NkSPt+rvAhTDZNsc60RGxmkjBoolBsQvXSyo3zERACgwFKW2RQkn7E35CbWiwvS2zZvUWnqBwRT1ZV1XZOVeu9K2P2bNtgr/5Zpv03dO5m3RbWkzE5s2zSqjd3SZ427bZT1WVWSh79/prQFOnmuBs326fO0EaNcqsM8Bchy+91Dvrg9vcCvhReLNmAdde23viPukk65dbh9m/34SutdWEykXOuSJ87vps2OCfp6fHtzpHjTLX4vr15hKdMsV3p23bZhZhS4tdj0mT7HqMG9e3cF42wm7AiRP974bdYmmqBEtIERjcgpS2heJc+xN+Qo0SnkyuFbdI79xuxx9vE/Qbb5j7aMSI3k/n775r3zvsMJuUX3nF1qvGj7fJ8uWXzS0H+GOoqDARaWszK6ylxT475BCzRs45x6yqPXss3NwlN+3pse9PmmRrYJs3m5hNnWr55YLRbFVV1n+Xj85ZZS6YYMwYO0dzs1lIbo0NsIi2X/3KzgkAZ57pb4rduNGEacsWE5zXXjOLaPNmE6rdu21NzEXuRUUxZiPTWgzdYmSQMbgFKW2RQo2NfkqfqLxomcg0ecW5VoLna283IRo+3CbpOXNMrIJP54CttWzdam6uV1+14+vqrP1Nmyx4QNU+q6y0tZbDDwe++EU75rbbTAhddoctW0zcFiwwq2z+fFu32bPHxGzRIlsXO/RQP2VPUIycqI4YYf0aOdIvK97aau91dVk/pk41UbzoIv96LFkCPPigBRVs2uRH5M2caW26IIoFC4Dnn7d+b9pka1CrVpnoTZ5sY851fcdZw+Es4HF7zAgZ4AxuQUpbpFBVlU3kzrqIWo/IRK6Tlztfe7u9njPHTyZaX2+up7DALVsGXHONTcB799o6y9at/l6b9eut1EJ1tf0W8UVk1SpzmX3848ALL9jxc+aYxbF5s1lCZ51lovANPk1nAAAVs0lEQVTWW7YutW6dneuSS6JdVsE8ftu3m4gB9t6ePX4evwMHTHxnzuwtaICJ0pIl5vZcvdqsns5Oc0O6fVrd3dbfmTOtr62tJkoXXeS3l4slE0yI64TWiVGarHZCSsjgFqS0uUQ6O239xlksSfbwFOJ8IrYBtqenb92kqH01Lvx70yb7PXmyL2SAWTlz5/at/eMeAFpbzRXX3m5WUV2db7V8+9uWVWLfPjv2Ix+x44HoLBTBh4pJk6z/GzdaUEJFhYmZC5KYMKF35oUgztJye7ZmzzZRPf98W1979VV/ferkk/0IxLAlGm4z7m/LWacbNtjvm2/2w9HTZLUTUkIGtyAB6XCJuImrqsom1O7u6A2shaa21j/fqaf6T/uZrof7zrHH2gL8Rz/q11gKJ3cN4x4AGhps749b+3EW1PLlto7kovN27zY34ujRfmbs/ft7i0H4oQKw9n/wAwv73rXL3H3d3cAXvtDXOnI4gdizx9x7I0f6YjZsmIWXA34m9Wx/M1EWUPDctbX2fjg3XtqsdkJKCAWp3ERVeY2rKlqo8wWf2nO1EKMEIGodJNP3Xai5swI6O61fy5fbetKOHSYiJ50EfNIrt/X443Z9nnvOgiuCodtRDxWPPOLvhTriCLOeFi2K75dzX+7fb5F227aZ5Td3rq2bZRPbMHEWkPtuTU10bry0We2ElBAKUrkJu2g6O/1Nm4Umbn0i10nPfSff9Y4oK8CJ1IUXWrReXR1wyim+VbF8uYnR7t0WqTdhQrQ7y+Xu6+62iLkJE3rXh4orcBd0l27bBpx3ni8+UWtpScYYZQGFs2SEs0YEry8hgwwKUrFIup8oXxdNrvunsiVZzYewmIYrqmbqb5QVEFxf2ry59ybWpUv9PUDt7TbZh69VXOVcJ0aZxDPovpw8ubcllG1ja1xkY9ACGjq0d4mJ4PeK9QBCyEEGBakY5GI55OOiydUyiZuo+7s+ERTT/fvjK6AmscyC18HtFwov7Le2WuBFV5fVXwoTjLgDeq/3hLMe5JviJ0i2++AsILdm5gR22bJk1V8JGWSwYmwxCFoOSapx1tT0zQBdyPaDE/Xs2TZRF2ISdJP45Zf7FVVF/D1UufbXXYf6+r5WY1OTWS7vfa+t96xa1bsK66pV9n5U5VwgmSVajPtQU2PBH27NrLvbT0OU9P4RMkighVQMih0plWv7weNzWZhPgrN0Ghri91Dl2t+owIlHHjE3ngsHdymSGhrM6nDnXbLEwtHDpSCi2oxbT0pK0nGFjwsGSjCSjpC/IZqkzHTKqaur0zVr1uTfQDHy2RU7R14u7Tc19S1hXqjzV1VZQEBzsyUbdXuorrrKd48Fo/Hi+ptpPKtWWZ6+sWMt4KGiwo4ZOhRYuNB377nKrOPHW+h2VBh7U5NfViPKvZhprJnWxtx1SDq+tOVQJKREiMhLqloX9RktpGLsjE862fRnUkoaidXUBHzlK5a0FLASFHfc0b8xhtekZs/289apmhVWVdX3usYt3me7B8ENtUcdZWswTU3+dXDWx759fkn1xx+38O1geLg7j8tFd8EF0RnQ4/oW3gPlzg9k/xuKWjOjEBHSCwpSLpFiSXAC4MJ977wz/om5FCli3H4YVxE2yQSczRpwNYBGjvR/q/Yuj5BLxoEkxy5caL+ddRcMCnB7oKqq7P21a+2YcOZzd55Zs0yQ1q61dEWZXGbuO3Eil7T/Sa4vIYMcClLQv9/RYdFQw4fnLxINDWaNVFfbpBcsgR2kVCliXDjz+vX2euzY+Ak4k0gGP+voMAFyGbnb2+0cYfdY0nWjTGsx4T7V12feu+Uq3S5f3jfzedDSikv/E9e3N96w11F1pcLRhsHw7qTXlxBCQUoUblwMSpUipqbGrLSGBnvtJuCoJ/VMItnQYK4uZ3U4ayhu7SSXMOpMx0b1KdO1c64wJ1zh9hYutI21rmRGtvUc17dgOfe4MhHumHARwExjyXU9jZABDAUJ6J154Kmn+icS9fWWZcBVco0LIMhn30u+hNcr4p7U4yb6piabaNevt5+TT05WLC6Xda646xDVpyTXLm7MwXWvpHuCsomcO6ax0QIl4gQnyUMIrSgyiKEgBSmESNTUWNBAUsugHJNN3JN63PgbGmztZP58y869dGn+/Y6KNrvuOl/AwwEXUX3Kx4JwYw6ue4X3BEWJSC61ibIJTpLQ81K5cglJIRSkMIUQibRHUCVxeTnirKN8iHr6b2iwchPV1ZaINGrNLdin/uTP6+iw2k2q/rrX3LmWI6+hoW/2ilzOFVdsL0zQGs/FSiVkEEBBGozkYgk6N9QFF1hUWn+so6in/0K0kbQ/qhYWfuSR5rL70IcsCGLfPivkFy6ImMuaT64imauVSsgggII0WElqxQUj06ZN69+m2qin/9pas7pcmHy29vO1IBobLXpy8mQLOqiosEJ+c+fa3qzqavsdtNCSnisfkczFSiVkkEBBIpkp5BN7XFt33pm8/WAbVVW+lZVUAMLh29u25d7fuLZzEUlaQoT0gamDSG6kJSQ5HzdZVNqgZcuAu+7KvpE5SX/ScF0ISTlMHUQKQ5QIAKWZiMMTfj5usrjw7SgLLVeBoZuNkH5DQSLJiUqzFMy0Xaw9M1FCmNRNFiUsYfFIuk+LEFJUKEikN7lsUgVKs2cmyho644zsazD5Cgv3AhFSFihIxCfbBB61sbO/mS2SEGcNZXOT5Sss3AtESFmgIBGfJBN4WARKESmWb0RavsLCCDhCygIFifjkG77sMg/0twJrkvPk+p18hYVBCoSUHAoS8cl3Ak9TEEBU7jkKCyEHBRQk0pt8JvC0BAGkSRgJITkzpNwdIAOAtAQBBIWxuzu/XHlxOJekK51OCCk4tJBI/0lLEECxhJGWFyElgYJECkM2V59b24mrMFuoPhRDGNPikiRkgENBIsUnqlrr6NHFsTSKEcSQFpckIQOc1AiSiKwAcAqAbu+tzap6TPl6RApGXLXWg8XSSItLkpABTmoEyeNKVf1RuTtBCoyzMHbtst+uWuvBZGkwfJyQopM2QSIDkXANo2KtIRFCDmrSJkjfEpHbAbwJ4GuquiLuQBH5HIDPAcD06dNL0zuSP7QwCCFZSNM+pOsBHAFgKoD7ADwmIkfGHayq96lqnarWTZgwoVR9JIQQUiRKIkgiskJENObnOQBQ1QZVbVPVDlX9KYDnAZxfiv4RQggpPyVx2anqmfl8DYAUuCuEEEJSSipcdiIyRkTOFZHhIjJURD4BYB6A35W7b4QQQkpDWoIaKgHcAmAWgB4AbwC4UFXfLGuvCCGElIxUCJKqbgcwt9z9IIQQUj5S4bIjhBBCKEiEEEJSAQWJEEJIKqAgEUIISQUUJEIIIamAgkQIISQVUJAIIYSkAgoSIYSQVEBBIoQQkgooSIQQQlIBBYkQQkgqoCARQghJBRQkQgghqYCCRAghJBVQkAghhKQCChIhhJBUQEEihBCSCihIJL00NQGrVtlvQsiAJxUlzAnpQ1MTcNNNQHc3MHQo8PWvAzU15e4VIaSI0EIi6aSx0cSottZ+NzaWuUOEkGJDQSLppLbWLKPGRvtdW1vmDhFCig1ddiSd1NSYm66x0cSI7jpCBjwUJJJeamooRIQMIuiyIwc3jMQjZMBAC4kcvDASj5ABBS0kcvDCSDxCBhSiquXuQ78Rke0ANpa7HwVkPICWcneiyPR7jMOAyqnAFAFEAd0MbOkAugrUv0LA+zgw4BgLywxVnRD1wYAQpIGGiKxR1bpy96OYcIwDA45xYJCWMdJlRwghJBVQkAghhKQCClI6ua/cHSgBHOPAgGMcGKRijFxDIoQQkgpoIRFCCEkFFCRCCCGpgIJUBkRknIg8KiJ7RGSjiFya5fhrRORdEdklIv8uIsMCn60Qkf0i0u79vFn8EcT2M/G4sowpp+tTSgo4xtTctyBJxyciJ4jI70SkRUT6+P0Hwj1MMMZU3kMgpzF+SkReEpHdIvKOiNwhIkNzbadgqCp/SvwD4CEADwMYCeB0ALsAHB9z7LkAmgEcD2AsgBUAbg98vgLAZ8s9plzGlWBMia/PQTzG1Ny3PMd3DIDLAXzYppH8/8YP4jGm8h7mOMbPAzgDQBWAqQBeAvBP5bqPZb9wg+0HQDWATgBHB957IDhZhY7/OYDbAq/PBvBu4HUq/lPkMq5MY8r1+hyMY0zTfct3fIHPjwpP1gPlHmYaY1rvYX+vP4AvAXisXPeRLrvSczSAHlVdF3jvFdiTdBTHe58Hj50oIocG3vuW51Z4XkTOLGhvk5PLuDKNKdfrU0oKNUZHGu5bkEJd+4FyD5OQtnsI9G+M8wC8VoB28oKCVHpGwszeILsAvCfh8e7f7vjrARwBM7fvA/CYiBxZmK7mRC7jyjSmXK9PKSnUGIH03Lcghbr2A+UeZiON9xDIc4wi8mkAdQD+pT/t9AcKUoHxFjo15uc5AO0ARoW+NgpAW0yT4ePdv9sAQFUbVLVNVTtU9acAngdwfuFGlJhcxpVpTLlen1JSqDGm6b4FKdS1Hyj3MCMpvYdAHmMUkQsB3A7gPFV1SVZLfh8pSAVGVc9UVYn5OR3AOgBDRWRm4GsnwjeTw7zmfR48tllVd8R1AYD0dxx5kMu4Mo0p1+tTSgo1xijKdd+CFOraD5R7mCtpuIdAjmMUkYUAfghgsar+Od92CkK5F+AG4w+AX8CiV6oBfACZo+wWAngXwHGwaK0/wltUBDAGFs01HFZs8RMA9gA4Js3jyjSmXK/PwTjGtN23PMcnXv+Pg03EwwEMG2D3MHaMab6HOY7xLAA7AMzrTzsF63e5L9xg/AEwDsCvvT/gTQAuDXw2HWYqTw+89yVYCPFuAPcH/lNMALAaZkLvBPA/AM5J27hyGVO261Pun0KMMW33LZ/xAaj1JungT+NAuoeZxpjme5jjGJ8B0O29537+q1z3kbnsCCGEpAKuIRFCCEkFFCRCCCGpgIJECCEkFVCQCCGEpAIKEiGEkFRAQSKEEJIKKEikaIjI90XkxsDrz4tIs1c75lAR+YCIvOW9vrCcfc2X8BgHIiLSKCILyt0PMvDhPiSSFyLSCGAibFNdD4DXAfwMwH2qeiDi+ErYBtFTVPUV773/BvBbVf1Oqfo9UBCRFQAeVNUfleBcjbAyC08X8Ry1AN4GUKmq3Wlrj5QGWkikPyxW1fcAmAFLzHg9gB/HHDsRlmYlmAdrBvLMixWsakmKA68xKTUUJNJvVHWXqv4WwCUAPiUiJwCAiPxERG4RkaMBuPLOO0XkjyKyAZa6/zHPZTdMREaLyI9FZKuIbPa+W+G1dZlXc+YuEfkrgG94739GRNaKSKtXbnqG65eXYf0Kzy3YKiLfExEJfL7M+26biLwuIid5708RkUdEZLuIvC0iV8WN3Y3R+/eZXhnoL4vINm8cn87w3RUicrM3rjYR+b2IjA98foqIvCAiO0XkFVdvR0RuhVX5vMe7dveIyE0icrf3eaVXcvoO7/UIsVLbY73XS0TkNa/dFSJybOCcjSJyvYi8CmBPWJREZJZ3TT4WM6bTRGS1WNn21SJyWqjtBYHX3xCRB72Xz3q/d3pjOjVwz+/22ntDRM7uR3tHichKr60WEXk47t6Q8kBBIgVDVf8E4B3YZBl8fx38ol5jVPUsVT0SlhtrsaqOVNUOAD+FuQCPAjAHwIcAfDbQVD2AvwA4DMCt3rrTDQD+DpZbbBUsEWSQCwDMhWUp/igsISZE5GKYqH0SllJ/CYAdIjIEwGOwQmRTYZVerxaRcxNehkkARnvfvRzA95wQxHApgE97Y6oCcK3Xv6kAngBwCyyf2LUAHhGRCar6NW+sV3rX7koAKwGc6bU5F5bYdb73+lQAb6pqq/dw8BCAq2HX7EnYQ0FVoE8fB7AIdq/+5u7yBPv3AL6oqr8ID0RExnl9/i6AQwH8G4AnpHdRwjjmeb/HeGN60Xvt7vl4AF8H8J/eefJp72av/2MBTANwd4J2SAmhIJFCswU2geaEiEwEcB6Aq1V1j6puA3AXgOCT+BZVvVtVu1V1H4B/APAtVV3rTZy3AXhf0EqCZdjeqaqbYIkk3+e9/1kAd6jqajXWq+pG2GQ+QVW/qaqdqvoXWGr+SIsggi4A31TVLlV9Epas8pgMx9+vquu88fwy0L+/B/Ckqj6pqgdU9Q8A1iC+3s6LAGZ6k/88mOt0qoiMhAnTSu+4SwA8oap/UNUuWDG2EQBOC7T1XVVt8vrkOAPAbwF8SlUfj+nDIgBvqeoD3j16CMAbABZnGH82tgH4tnc9H4ZZ2ovybKsL5iaeoqr7VfW5fvSLFAEKEik0UwH8NY/vzQBQCWCr50raCeAHMMvB0RTxne8Ejv8rrGTA1MAx7wb+vRdWBRMAagBsiOnHFNem1+4NsDWwJOwILaIHzxlFXP9mALg41I/TAUyOasQTjzUw8ZkHE6AXYCUDgoI0BcDGwPcOwK5r8JqFrzMAXAHgBVV9JsNYerXtsTHUdq5s1t6RVxu98+TDdbC/jz95LsvP9KNfpAhw0ZIUDBGZC5t88nnybALQAWB8hqiocEhoE4BbVfU/8jxfVLnpJgBvq+rMiM9KSROAB1R1WcznUeGxK2H1bebASiOshLko3w9/TWULgPe6L3hrajUANmdp+woA14vIXap6TUyftsCENMh0AE95/94D4JDAZ5OynBMwK08CojQdZqnl3J6qvgtgGQCIyOkAnhaRZ1V1fcy5SYmhhUT6jYiMEpELYMW8HtTeVScToapbYf79f/XaGyIiR4rI/Axf+z6Ar4rI8V4/RntrQ0n4EYBrReRkMY7yXH1/ArDbW9gfISIVInKCJ7al5EEAi0XkXK8Pw8WCJqZ5nzfDgkKCrIStib2uqp0AVsBck2+r6nbvmF8CWCQiZ4uF4n8Z9iDwQpb+tMGKDs4TkdtjjnkSwNEicqmIDBWRS2DF7ZyL738BfMwLuqgDsDTw3e0ADkSM6TAAV3nfuRjAsd55cm5PRC4OXL9WmGj1ZBk3KSEUJNIfHhORNtjT/Ndgi9ixUWUJ+CRsYf912ISxHDEuKgBQ1UcB/DOAX4jIbgD/B1uHyoqq/grArQB+Dptsfw1gnKr2wNY83gfbx9ICE6/R+Q0pP1S1CcCHYe7C7bBr/BX4/2e/A2CpWPTgd733XoCtBzlr6HUA+wOvoapvwtan7oaNbTEssKQzQZ92AjgHwHkicnPE5ztgQSRfhlUhvQ7ABara4h1yI8wqbQVwE+zau+/uhd2P5z0X5SneRw0AZnp9vRXAUvXLwOfa3lwADSLSDrOy/lFV3842blI6uDGWEJJKROQy2Ibc08vdF1IaaCERQghJBRQkQgghqYAuO0IIIamAFhIhhJBUQEEihBCSCihIhBBCUgEFiRBCSCqgIBFCCEkFFCRCCCGp4P8BJCnCnGSCNG8AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x324 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "column = \"scan_sm_log_change\"\n",
    "\n",
    "ddg_change = results_df[column].values\n",
    "ddg_exp = results_df[\"ddg_exp\"].values\n",
    "textstr = \"\"\"\\\n",
    "Pearson R: {:.3f} (p={:.0e})\n",
    "Spearman R: {:.3f} (p={:.0e})\n",
    "\"\"\".format(\n",
    "    *stats.pearsonr(-ddg_change, ddg_exp), *stats.spearmanr(-ddg_change, ddg_exp)\n",
    ").strip()\n",
    "props = dict(facecolor=\"white\", alpha=0.3)\n",
    "\n",
    "with plt.rc_context(rc={\"font.size\": 12}):\n",
    "    fg, ax = plt.subplots(figsize=(6, 4.5))\n",
    "    ax.plot(-ddg_change, ddg_exp, \"r.\", alpha=0.5)\n",
    "    ax.set_xlabel(\"Difference in network outputs\")\n",
    "    ax.set_ylabel(\"ΔΔG (kcal / mol)\")\n",
    "#     ax.set_xlim(-8, 16)\n",
    "    ax.set_ylim(-8, 16)\n",
    "    ax.text(0.04, 0.95, textstr, transform=ax.transAxes, fontsize=\"medium\", verticalalignment=\"top\", bbox=props)\n",
    "fg.tight_layout()\n",
    "# fg.savefig(NOTEBOOK_PATH.joinpath(f\"{UNIQUE_ID}-protherm-all.svg\"), bbox_inches=\"tight\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Timing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "def design_sequences(structure, num_sequences):\n",
    "    chain_id = next(structure.chains).id\n",
    "    pdata = proteinsolver.utils.extract_seq_and_adj(structure, chain_id)\n",
    "    data = proteinsolver.datasets.protein.row_to_data(pdata)\n",
    "    data = proteinsolver.datasets.protein.transform_edge_attr(data)\n",
    "    data.x = torch.ones_like(data.x) * 20\n",
    "    data = data.to(device)\n",
    "    for _ in range(num_sequences):\n",
    "        _ = proteinsolver.utils.design_sequence(net, data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5vli02.pdb 1\n",
      "CPU times: user 165 ms, sys: 4.02 ms, total: 169 ms\n",
      "Wall time: 168 ms\n",
      "\n",
      "5vli02.pdb 1000\n",
      "CPU times: user 1min 36s, sys: 422 ms, total: 1min 36s\n",
      "Wall time: 1min 36s\n",
      "\n",
      "1n5uA03.pdb 1\n",
      "CPU times: user 393 ms, sys: 8.02 ms, total: 401 ms\n",
      "Wall time: 400 ms\n",
      "\n",
      "1n5uA03.pdb 1000\n",
      "CPU times: user 3min 47s, sys: 1.1 s, total: 3min 48s\n",
      "Wall time: 3min 48s\n",
      "\n",
      "4z8jA00.pdb 1\n",
      "CPU times: user 1.02 s, sys: 24 ms, total: 1.04 s\n",
      "Wall time: 1.04 s\n",
      "\n",
      "4z8jA00.pdb 1000\n",
      "CPU times: user 3min 57s, sys: 1.28 s, total: 3min 59s\n",
      "Wall time: 3min 59s\n",
      "\n",
      "4unuA00.pdb 1\n",
      "CPU times: user 556 ms, sys: 0 ns, total: 556 ms\n",
      "Wall time: 555 ms\n",
      "\n",
      "4unuA00.pdb 1000\n",
      "CPU times: user 4min 29s, sys: 1.38 s, total: 4min 31s\n",
      "Wall time: 4min 31s\n",
      "\n",
      "4beuA02.pdb 1\n",
      "CPU times: user 2.42 s, sys: 100 ms, total: 2.52 s\n",
      "Wall time: 2.52 s\n",
      "\n",
      "4beuA02.pdb 1000\n",
      "CPU times: user 12min 36s, sys: 1min 12s, total: 13min 48s\n",
      "Wall time: 13min 48s\n",
      "\n"
     ]
    }
   ],
   "source": [
    "structure_path = Path(proteinsolver.__path__[0]).joinpath(\"data\", \"inputs\").resolve(strict=True)\n",
    "for structure_id in [\"5VLI\", \"1N5U\", \"4Z8J\", \"4UNU\", \"4BEU\"]:\n",
    "    structure_files = list(structure_path.glob(f\"{structure_id.lower()}*.pdb\"))\n",
    "    assert len(structure_files) == 1, structure_files\n",
    "    structure_file = structure_files[0]\n",
    "    structure = PDB.load(structure_file)\n",
    "    for num_designs in [1, 1000]:\n",
    "        print(structure_file.name, num_designs)\n",
    "        %time design_sequences(structure, num_designs)\n",
    "        print()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Perplexities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calculate_perplexity(net, x, edge_index, edge_attr, x_in=None):\n",
    "    if x_in is None:\n",
    "        x_in = torch.ones_like(x) * 20\n",
    "\n",
    "    mask = x_in == 20\n",
    "\n",
    "    if not mask.any().item():\n",
    "        return []\n",
    "\n",
    "    output = net(x_in, edge_index, edge_attr)\n",
    "    output = torch.softmax(output, dim=1)\n",
    "\n",
    "    probas = output.gather(1, x.view(-1, 1))\n",
    "    probas[~mask] = -1\n",
    "\n",
    "    max_proba, max_residue = probas.max(dim=0)\n",
    "    x_in[max_residue] = x[max_residue]\n",
    "\n",
    "    return [max_proba.item()] + calculate_perplexity(net, x, edge_index, edge_attr, x_in=x_in)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/strokach/env/lib/python3.7/site-packages/ipykernel_launcher.py:4: TqdmDeprecationWarning: This function will be removed in tqdm==5.0.0\n",
      "Please use `tqdm.notebook.tqdm` instead of `tqdm.tqdm_notebook`\n",
      "  after removing the cwd from sys.path.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "48e80c54611445c59263eb2860d69368",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1283), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "proba_lists = []\n",
    "for i, data in enumerate(\n",
    "    tqdm.tqdm_notebook(\n",
    "        DataLoader(datasets[\"protein_test\"], shuffle=False, num_workers=1, batch_size=1, drop_last=False)\n",
    "    )\n",
    "):\n",
    "    data = data.to(device)\n",
    "    with torch.no_grad():\n",
    "        proba_lst = calculate_perplexity(net, data.x, data.edge_index, data.edge_attr)\n",
    "    assert len(proba_lst) == data.x.size(0)\n",
    "    proba_lists.append(proba_lst)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8.974993654617126\n"
     ]
    }
   ],
   "source": [
    "perplexities = [2 ** -(np.log2(proba_lst).mean()) for proba_lst in proba_lists]\n",
    "print(np.mean(perplexities))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAVQAAAEPCAYAAAAd2zyBAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAd6UlEQVR4nO3deZhdVZnv8e+PBEwglAgpUAJJBISk4zWI5dC2IjaTgNJcIrdRkUEkCk+k+4JXsC9DmLoBjV6vUSAIJAwqYgdsCIjyGGTwMhStAaORNpBImEwClBkgYXjvH2tVsjlUpXbMPlPy+zzPeerstfY5+z2Hk5e11157LUUEZma24TZrdgBmZhsLJ1Qzs4o4oZqZVcQJ1cysIk6oZmYVcUI1M6uIE6qZWUUallAlTZLULWmVpOk1dVtK+q6kJZJ6JN1VqJOkiyQtzY+LJalRcZuZlTW4gcd6CjgfOBAYWlM3LccyFngO2LNQNxE4DBgPBPBz4DHg0nUdbPjw4TF69Ogq4jYzW+Ohhx5aEhGdfdU1LKFGxEwASV3ATr3lkvYADgV2ioi/5OKHCi89BpgSEYvy/lOAExggoY4ePZru7u7qPoCZGSBpYX91rdCH+n5gIXBOPuV/RNKEQv04YE5he04uMzNrKa2QUHcC3gn0ADsCk4AZksbm+mG5rlcPMKyvflRJE3M/bffixYvrHLaZ2eu1QkJ9EXgZOD8iVkfEL4HZwAG5fjnQUdi/A1gefczqEhHTIqIrIro6O/vs4jAzq5tWSKgPD1A/l3RBqtf4XGZm1lIaOWxqsKQhwCBgkKQhkgYDdwF/Ar6a9/k7YB/g9vzSq4FTJI2QtCNwKjC9UXGbmZXVyBbqGaTT+9OBo/LzMyLiZeAfgINJ/aOXA0dHxLz8usuAm4FHgN8Cs3KZmVlL0cY6wXRXV1d42JSZVU3SQxHR1VddK/ShmpltFJxQzcwq0shbT62k0afPWmf9ggsPaVAkZrY+3EI1M6uIE6qZWUWcUM3MKuKEamZWESdUM7OKOKGamVXECdXMrCIeh9qGPE7VrDW5hWpmVhEnVDOzijihmplVxAnVzKwiTqhmZhVxQjUzq0gj15SalJd4XiVpej/7nC0pJO1XKJOkiyQtzY+L+1pC2sys2Ro5DvUp4HzgQGBobaWkXYFPAk/XVE0EDiOtdhrAz4HHgEvrGayZ2fpqWAs1ImZGxE3A0n52mQqcBqyuKT8GmBIRiyLiSWAKcGzdAjUz+yu1RB+qpCOA1RFxax/V44A5he05uczMrKU0/dZTScOAfwUO6GeXYaTlpXv1AMMkKWqWbJU0kdRFwMiRI+sQrZlZ/1qhhXoOcE1EPN5P/XKgo7DdASyvTaYAETEtIroioquzs7MOoZqZ9a8VEuq+wMmSnpH0DLAz8CNJp+X6uaQLUr3G5zIzs5bSsFN+SYPz8QYBgyQNAV4hJdTNC7s+CJwC3Ja3rwZOkXQr6Sr/qcC3GxW3mVlZjexDPQM4u7B9FHBOREwu7iTpVeD5iFieiy4DdgEeydvfy2W2ATwFoFn1GpZQc+KcXGK/0TXbAXwlP8zMWlYr9KGamW0UnFDNzCrihGpmVhEnVDOzijihmplVxAnVzKwiTqhmZhVxQjUzq0jTZ5uy6g10F5SZ1cdf1UKVNFTSFlUHY2bWzkolVElfl/S+/PwQ4DngBUmfqGdwZmbtpGwL9TPAb/Pzs0gTmxxKmhjazMwo34e6ZUSslLQdsEtE/DuApFH1C83MrL2UTaiPSvoMsBtp1VEkDQderFdgZmbtpmxCPQn4FvAy8LlcdiDws3oEZWbWjkol1Ih4EPhgTdl1wHX1CMrMrB2VHjYlaX9JV0i6OW93Sfr7+oVmZtZeyg6b+hJwCfBfwN65+EXg/LIHkjRJUrekVZKmF8o/IOnnkp6TtFjSDZLeVqiXpIskLc2PiyWp7HHNzBqlbAv1n4H9IuJC4LVcNg/YYz2O9RQpAV9ZU/4WYBowGhgFLAOuKtRPBA4jrXb6LuDjwBfW47hmZg1R9qLU1sAT+Xnkv5sDq8seKCJmQuoqAHYqlN9W3E/SVOCXhaJjgCkRsSjXTwFOAC4te2wzs0Yo20K9Czi9puxkYHa14QCpS2FuYXscMKewPSeXvYGkiblboXvx4sV1CM3MrH9lE+qXgP8uaQGwtaQ/AEcAp1QZjKR3ke7E+l+F4mFAT2G7BxjWVz9qREyLiK6I6Ors7KwyNDOzAZUdNvW0pPcC7yX1cz4BPBARr637leVJ2g24DfiniLi7ULUc6ChsdwDL8/LSZmYto1RClbQnsDQiHgAeyGU7S9o2Iuas+9Wl3n8UcAdwXkRcU1M9l3RB6oG8PZ7XdwmYmbWEsqf815IuQhVtAdQmv35JGixpCDAIGCRpSC4bAfwC+E5E9HWh6WrgFEkjJO0InApML3tcM7NGKXuVf2REPFYsiIj5kkavx7HOAM4ubB8FnEMaNbALcLakNfURMSw/vSzXP5K3v5fLzMxaStmEukjSXhHxn70FkvYijS0tJSImA5P7qT5nHa8L4Cv5YWbWssom1G8CP5F0MTAf2BX4MnBBvQIzM2s3Za/yXy7pBeB4YGfSVf5TI+LH9QzOzKydlF6kLyJuAG6oYyxmZm2tdEKVdACwJ2mg/RoRcVbVQZmZtaOy41CnAv+DdKvpykKVB9ebmWVlW6ifAvaMiCcG3NPMbBNVdmD/UuCFegZiZtbuyrZQpwDXSfo34NliRe2AfzOzTVXZhHpJ/vvxmvIg3UpqZrbJKzsOtfTaU2Zmm6r1SpR5hqkP1CsYM7N2VnaRvpGS7iWtI3VHLvukpO/VMzgzs3ZStoV6GTCLtLbUy7ns58D+9QjKzKwdlb0o9T7gkIh4TVIARESPpDfXLzQzs/ZStoX6LLBbsUDS3wB/qjwiM7M2VbaF+nXgljwOdbCkTwH/AlxYt8isqUafPmvAfRZceEgDIjFrH2WHTV0p6TlgImnqvmOAMyPipnoGZ2bWTkoPm4qImyLi4IgYFxEfW99kKmmSpG5JqyRNr6nbV9I8SSslzc6L9vXWSdJFkpbmx8V9LSFtZtZsZWeb+lx/dRFxZcljPQWcDxwIDC2893BgJvB54GbgPOB6oHe860TgMNJqp0EaXfAY0NeCfmZmTVO2D/WzNdtvJS2Dci9QKqFGxEwASV3AToWqw4G5eQJrJE0GlkgaExHzSN0LUyJiUa6fApyAE2rTDdTP6j5W29SU7UP9aG1ZbrWOrSCGccCcwrFWSJqfy+fV1ufn4/p6I0kTSS1aRo4cWUFoZmblbcg9+tNJa0xtqGFAT01ZD+kmgr7qe4BhffWjRsS0iOiKiK7Ozs4KQjMzK69sH2pt4t0SOIpq5khdDnTUlHUAy/qp7wCW5+WlzcxaRtk+1Fd443InT5L6MjfUXFI/KQCStiL1z84t1I8HHsjb4wt1ZmYto2xCfXvN9oqIWLI+B5I0OB9vEDBI0hBSor4R+JqkCaT5As4CHs4XpACuBk6RdCspqZ8KfHt9jm1m1ghlL0otrOBYZwBnF7aPAs6JiMk5mU4FrgXuB44s7HcZsAvwSN7+Xi4zM2spZftQ76bECqcRsfc66iYDk/upuwMY009dAF/JDzOzllX2lH828DlgBrAQGEnq97ySNMjezGyTVzahHgAcGBFrLgZJ+j5wZUR4Bn8zM8qPQx0LzK8pe5x+TtPNzDZFZRPqL4Hpkt4haaik3YErgLvrF5qZWXspm1CPzX/nAitIV9wFHFeHmMzM2lLZYVPPAUfmO6Y6gcUR8VpdIzMzazNlL0ohaSzwSWCHiJgkaQ/gTRHxcN2iMzNrI2WXkT4CuAsYARydi7cGvlGnuMzM2k7ZPtRzgf0j4ovAq7lsDum+ejMzo3xC3Z61c5JG4a9nfDIzy8om1Id446z9R7J2Bigzs01e2YtSJwM/k3Q8sJWk24HdSXdQmZkZ5YdNzZM0Bvg4cAtpKelbImJ5PYMzM2snpYdNRcRK4EcAknYBtiPNpm/rYaCF7cysfZUdNvUDSR/Mz48j3TH1u9wFYGZmlL8otS/QnZ+fAuwHvA84vR5BmZm1o7Kn/FtExGpJI4BtI+JeAEk71C80M7P2UraF+htJXwXOJK37RE6uf6kqEEmjJd0q6XlJz0iamtehQtK+kuZJWilptqRRVR3XzKwqZRPq8cB/A4aS1oYC+Fvgugpj+S7wZ+BtwJ7AR4CTJA0HZpKS+bakrofrKzyumVklyg6bmg98uqbsx8CPK4zl7cDUiHgJeEbST4FxwOHA3Ii4AUDSZGCJpDGFlVHNzJqubAu1Eb5FmiJwy9ydcBDQm1R7b3slIlaQVg8YV/sGkiZK6pbUvXjx4gaFbWaWtFJC/SUpSf4FWEQ6tb8JGAb01OzbQ5rt6nUiYlpEdEVEV2dnZ53DNTN7vZZIqHni6ttJfaVbAcOBtwAXkW4e6Kh5SQewrJExmpkNpN+EKum+wvOz6xzHtsDOpD7UVRGxFLgKOJh0E8GaaQIlbQXsmsvNzFrGulqou0sakp+fWs8gImIJaRXVEyUNlrQNcAyp7/RG4J2SJuR4zgIe9gUpM2s167rK/xPgUUkLgKGS7uprp4jYu6JYDgf+D3AaaRLr2cD/jIjFkiYAU4FrgftJUweambWUfhNqRBwn6UPAaOC9pGWj6yYifgPs00/dHcCYeh7fzGxDrXMcakTcA9wjaYuImNGgmMzM2lLZgf1XSvooadb+EcCTwLUR8Yt6BmftrcxUhQsuPKQBkZg1Rtnp+z5Put3zGdLQpqeB70s6oY6xmZm1lbKzTX2FtOrpmjuWJF0P/DtweT0CMzNrN2UH9m8H/K6m7A+k8aNmZkb5hHoP8A1JW8KawfVfA35Vr8DMzNpN2YT6ReBdQI+kZ4EXSHcvfaFegZmZtZuyV/mfBj4iaSdgR+CpiFhU18jMzNpM6VVPAXISdSI1M+tDS8w2ZWa2MXBCNTOryIAJVdJmkv5e0haNCMjMrF0NmFAj4jXgJxGxugHxmJm1rbKn/HdJ+kBdIzEza3Nlr/IvBG6T9BPgCSB6KyLirHoEZmbWbsom1KGkBfMAdqpTLGZmba3swP7j6h2ImVm7Kz1sStJYSWdKmpq395D0riqDkXSkpN9LWiFpvqQP5/J9Jc2TtFLSbEmjqjyumVkVys6HegRwF2ly6aNz8dbAN6oKRNL+pGWjj8vvvTfwmKThpDlYzyTNbtVNmpvVzKyllG2hnkuaD/WLpAX0IK1IOr7/l6y3c4BzI+K+iHgtIp6MiCdJi/fNjYgbIuIlYDIwXpLXmDKzllI2oW5PSqCw9gp/FJ5vEEmDgC6gU9IfJS2SNFXSUGBc4dhExApgfi6vfZ+JkroldS9evLiK0MzMSiubUB8irSdVdCTwQEVx7ABsDnwS+DCwJ/Bu4AxgGNBTs38PqVvgdSJiWkR0RURXZ2dnRaGZmZVTdtjUycDPJB0PbCXpdmB34ICK4ngx//12nioQSd8gJdS7gI6a/TuAZRUd28ysEmWHTc3LfZYfB24hDe6/JSKWVxFERDwvaRF9dyHMBY7p3cirBeyay83MWkbpYVMRsRK4F7gTuLuqZFpwFfAlSdtLegvwz6TkfSPwTkkTJA0BzgIejoh5FR/fzGyDlB02NVLS3cACYBawQNI9FY8HPQ94EHgU+D3wa+CCiFgMTAAuAJ4H3k/qvzUzayll+1BnkC5MfSwiVkgaRkqAM4B9qggkIl4GTsqP2ro7AA+TMrOWVjahvgc4ICc9ImK5pNOApXWLzMyszZTtQ70PeF9NWRfw/6oNx8ysffXbQpV0bmFzPnCrpFmkK/w7AwcD369veGZm7WNdp/w712zPzH+3B1aRrr4PqUdQZmbtqN+E6in7zMzWT9mLUkjaEtiNdCvoGhHxq6qDMjNrR6USqqSjganAatbeJgrpzqaRdYjLzKztlG2hXgxMiIif1zMYM7N2VnbY1GrSLadmZtaPsgn1TOAbefZ8MzPrQ9mE+ihwKPCspFfz4zVJrw70QjOzTUXZPtRrgKtJazm9OMC+ZmabpLIJdTvgrIioZMkTM7ONUdlT/qt44xIoZmZWULaF+j5gkqT/DTxbrIiIvSuPysysDZVNqJfnh1lDjT591oD7LLjwkAZEYjawsmtKzah3IGZm7a7sraef668uIq6sKhhJ7wAeAX4cEUflsn2B75Bucb0fODYiFlZ1TDOzqpQ95a+9IPVW0sqj9wKVJVRS4nywdyPfSDAT+DxwM2nZleuBD1R4TDOzSpQ95f9obVlutY6tKhBJRwIvAL8izWoFcDgwNyJuyPtMBpZIGuNVT82s1ZReRroP04HjqwhCUgdwLnBqTdU4YE7vRkSsIK0eMK6f95koqVtS9+LFi6sIzcystLLLSG9W8xgGTCS1KKtwHnBFRDxRUz4M6Kkp6wG27utNImJaRHRFRFdnZ2dFoZmZlVO2D/UV0tynRU8CJ2xoAJL2BPYD3t1H9XKgo6asA1i2occ1M6ta2YT69prtFRGxpKIY9gFGA3+SBKlVOkjS3wCXAsf07ihpK9LFsLkVHdvMrDJlL0rVc5jSNOCHhe0vkxLsiXn7a5ImALOAs4CHfUHKzFrROhOqpNm88VS/KCJi3w0JICJWAisLx1wOvBQRi/P2BNLyK9eSxqEeuSHHMzOrl4FaqNf2Uz4COBnYstpwICIm12zfAYyp+jhmZlVbZ0KNiCuK25K2A75Kuhh1PWmok9lfrcy9+mbtouywqQ5J5wF/BHYA9oqIiRGxqK7RmZm1kXUmVElDJX0VeIx0V9SHIuKzETG/IdGZmbWRgfpQHwcGkZaR7gZ2kLRDcYeI+EWdYjMzaysDJdSXSFf5T+ynPoBdKo3IzKxNDXRRanSD4jCrG09SbY2yIZOjmJlZgROqmVlFnFDNzCrihGpmVhEnVDOzijihmplVxAnVzKwiTqhmZhVxQjUzq0jZJVDMWpanALRW0RItVElvknSFpIWSlkn6taSDCvX7SponaaWk2ZJGNTNeM7O+tERCJbWUnwA+ArwZOBP4kaTRkoYDM3PZtqRZr65vVqBmZv1piVP+iFgBTC4U3SLpceA9wHbA3Ii4AUDSZGCJpDFerM/MWklLJNRaec7V3UnLRZ8IzOmti4gVkuYD44CWS6juzzPbdLVcQpW0OXAdMCMi5kkaBiyu2a0H2LqP104EJgKMHDmy3qGardGoKQIHOo6nIWyulkqokjYDrgFWA5Ny8XKgo2bXDmBZ7esjYhowDaCrq2tdy1+brZcqzjycDDd+rXJRCkkCriAtAjghIl7OVXOB8YX9tgJ2zeVmZi2jlVqol5AWAtwvIl4slN8IfE3SBGAWcBbwsC9Imb2RVydorpZIqHlc6ReAVcAzqbEKwBci4rqcTKcC1wL3A0c2JVCzOvIFzfbXEgk1IhYCWkf9HcCYxkVktvFyX279tEwfqplZu2uJFqqZtQ73w/71nFDNbL2526BvTqhm+IKQVcN9qGZmFXFCNTOriBOqmVlFnFDNzCrihGpmVhFf5Tezym2qY1mdUM2sJbXjWFef8puZVcQJ1cysIk6oZmYVcUI1M6uIL0qZWVtqxZEEbqGamVWkLRKqpG0l3ShphaSFkj7d7JjMzGq1yyn/d0hLS+8A7AnMkjQnIhq68qmneDOrzsb476nlW6h52egJwJkRsTwi7gH+A/hscyMzM3u9lk+owO7AqxHxaKFsDjCuSfGYmfWpHU75hwE9NWU9wNa1O0qaCEzMm8sl/aHOsf21hgNLmh1Ei/B3sZa/i7Uq+S50UQWRvNGo/iraIaEuBzpqyjqAZbU7RsQ0YFojgtoQkrojoqvZcbQCfxdr+btYq12/i3Y45X8UGCzpHYWy8UBDL0iZmQ2k5RNqRKwAZgLnStpK0t8B/wBc09zIzMxer+UTanYSMBT4M/AD4MRGD5mqWMt3SzSQv4u1/F2s1ZbfhSKi2TGYmW0U2qWFambW8pxQzcwq4oTaYJLulPSSpOX50apjZSslaZKkbkmrJE2vqdtX0jxJKyXNltTvOL+NQX/fhaTRkqLw21gu6cwmhlpXkt4k6Yo8P8cySb+WdFChvu1+F06ozTEpIoblxx7NDqZBngLOB64sFkoaThrFcSawLdANXN/w6Bqrz++iYJvC7+O8BsbVaIOBJ4CPAG8m/QZ+lP/H0pa/i3YY2G8bgYiYCSCpC9ipUHU4MDcibsj1k4ElksZExLyGB9oA6/guNil5SOTkQtEtkh4H3gNsRxv+LtxCbY5/k7RE0r2S9ml2ME02jjQ3A7DmH9l8Nu25GhZKWiTpqtxS2yRI2oE0d8dc2vR34YTaeKcBuwAjSGPtbpa0a3NDaqrSczVsApYA7yXdK/4e0ndwXVMjahBJm5M+64zcAm3L34UTaoNFxP0RsSwiVkXEDOBe4OBmx9VEpedq2Njl6Sm7I+KViHgWmAQcIKn2+9moSNqMdOfjatJnhjb9XTihNl8AanYQTTSXNDcDsGb+213xXA2QfhuwEf8+JAm4gjR5/ISIeDlXteXvwgm1gSRtI+lASUMkDZb0GWBv4PZmx1Zv+fMOAQYBg3q/A+BG4J2SJuT6s4CHW/nCw4bq77uQ9H5Je0jaTNJ2wP8F7oyI2lPfjcklwFjgExHxYqG8PX8XEeFHgx5AJ/Ag6bTlBeA+YP9mx9Wgzz6Z1OIqPibnuv2AecCLwJ3A6GbH24zvAvgU8DiwAngauBp4a7PjreP3MCp/9pdIp/i9j8+06+/C9/KbmVXEp/xmZhVxQjUzq4gTqplZRZxQzcwq4oRqZlYRJ1Qzs4o4oVrbyXPKfr6C95nryWmsSk6oVglJCyS9mCdFfjbPlDSs2XGtS0SMi4g7IU0PJ+naJodkbc4J1ar0iYgYBuxFmjXpjPV9g3w7qtWQNKjZMdjAnFCtchHxJHAb8E4ASW/OS108LelJSef3JghJx+Z5Yb8p6TlgcqHs25J68jIY+/Z3PEmfk/R7Sc9Lur13qQxJH8zzzu6ct8dLekHSmLy9QNJ+kj4G/Avwj7mFPUfSEZIeqjnOqZJu6ieGYyU9lpfyeDzP09Bbd0KOb5mk30naK5ePzd0XL+Tuh0MLr5ku6RJJt0paAXxUacmQr0v6Uz4LuFTS0PX/L2T14oRqlcsJ7GDg17loBvAKsBvwbuAAoNgH+n7gMWB74IKasuHA2cBMSdv2cazDSMnwcNJcCXcDPwCIiF8BlwEzcuK5BjgjaibYiIifAv8KXB9p2ZHxwH8Ab5c0trDrUfk9amPYijSRyUERsTXwQeA3ue4I0n36R5OmnzsUWJrn/7wZ+Fn+3F8CrpNUXBLn0/n72Bq4B7iINAHznvm7HEGaNMRaRbMnE/Bj43gAC0gTW7wALAS+CwwlTcu2Chha2PdTwOz8/FjgTzXvdSxp3SUVyh4APpuf3wl8Pj+/DTi+sN9mwEpgVN7eHHgIeAT4ac17LgD2y88nA9fWxHEJcEF+Pg54HnhTH599q/y5JxQ/Z667HfinPl7zYeAZYLNC2Q9YO2HMdODqQp1Ik6bsWij7W+DxZv+392Ptwy1Uq9JhEbFNRIyKiJMiTcc2ipTUns6nti+QWo3bF173RB/v9WTkrJEtBHbsY79RwLcK7/0cKfmMAIg0v+Z0UvfDlJr3HMgM4NN5zs7PAj+KiFW1O0VanuMfgS/mzzmrt1sB2Jm0dEetHYEnIuK1ms84orBd/F46gS2Bhwqf9ae53FqEE6rV2xOkFurwnGy3iYiOiCiuDdRXkhuRE1mvkaRWa1/v/4XCe28TEUMjne4jaQSpy+AqYIqkN/UT5xtiiIj7SLPIf5h0+v2G0/3CvrdHxP7A20hTzl1eiK+vJW6eAnbOs9UXP+OT/cS0hDSN3bjC53xzpIuA1iKcUK2uIuJpUj/hFEkdefLkXSV9ZICXbg+cLGnz3A85Fri1j/0uBb4qaRysuQB2RH4uUuv0CuB40hyj/S3L/CwwuibBQZqTdCrwSkTc09cLJe0g6dDcl7qK1PXxaq7+HvBlSe9Rslu+aHY/6RT+K/kz7gN8AvhhX8fILdnLgW9K2j4fd4SkA/v5PNYETqjWCEcDWwC/I/VD/pjUkluX+4F3kFpmFwCfjIiltTtFxI2kizU/lPQX4LfAQbn6ZFIf7pn5VP844DhJH+7jeDfkv0sl/Weh/BpSd0G/rVPSv6NTSa3O50jrzJ+U47shx/990sTiNwHbRsRq0gWqg/Jn/C5wdKx7RvrTgD8C9+XPegewxzr2twbzBNPWciQdS7ro9KEWiGUo8Gdgr4j4r2bHY63NLVSzdTsReNDJ1MrwXSlm/ZC0gDRi4LAmh2Jtwqf8ZmYV8Sm/mVlFnFDNzCrihGpmVhEnVDOzijihmplVxAnVzKwi/x+lO0PBOLHkxAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 360x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fg, ax = plt.subplots(figsize=(5, 4))\n",
    "ax.hist(np.clip(perplexities, 3, 22), bins=30, range=(3, 22))\n",
    "ax.set_xlabel(\"Perplexity score\")\n",
    "ax.set_ylabel(\"Number of sequences\")\n",
    "fg.savefig(NOTEBOOK_PATH.joinpath(f\"{UNIQUE_ID}-perplexity.svg\"), bbox_inches=\"tight\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "ename": "Exception",
     "evalue": "Done!",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mException\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-43-2300d1589a50>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mException\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Done!\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mException\u001b[0m: Done!"
     ]
    }
   ],
   "source": [
    "raise Exception(\"Done!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results = []\n",
    "\n",
    "for i, data in enumerate(tqdm.tqdm_notebook(protherm_dataset, total=len(protherm_dataset))):\n",
    "    row = protherm_df.loc[data.Index.item()]\n",
    "    data = data.to(device)\n",
    "\n",
    "    with torch.no_grad():\n",
    "        log_prob_wt = proteinsolver.utils.get_node_proba(net, data.x, data.edge_index, data.edge_attr, 20)\n",
    "\n",
    "    wt_idx = torch.tensor(proteinsolver.utils.seq_to_tensor(row.mutation[0].encode()), dtype=torch.long, device=device)\n",
    "    mut_idx = torch.tensor(proteinsolver.utils.seq_to_tensor(row.mutation[-1].encode()), dtype=torch.long, device=device)\n",
    "    pos = int(row.mutation[1:-1])\n",
    "    x_mut = data.x.clone()\n",
    "    assert x_mut[pos - 1] == wt_idx\n",
    "    x_mut[pos - 1] = mut_idx\n",
    "    with torch.no_grad():\n",
    "        log_prob_mut = proteinsolver.utils.get_node_proba(net, x_mut, data.edge_index, data.edge_attr, 20)\n",
    "        \n",
    "    results.append(log_prob_mut.sum().item() - log_prob_wt.sum().item())\n",
    "        \n",
    "results_df[\"sum_log_prob_change\"] = results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "column = \"sum_log_prob_change\"\n",
    "ddg_change = results_df[column].values\n",
    "ddg_exp = results_df[\"ddg_exp\"].values\n",
    "textstr = \"\"\"\\\n",
    "Pearson R: {:.3f} (p={:.0e})\n",
    "Spearman R: {:.3f} (p={:.0e})\n",
    "\"\"\".format(\n",
    "    *stats.pearsonr(-ddg_change, ddg_exp), *stats.spearmanr(-ddg_change, ddg_exp)\n",
    ").strip()\n",
    "props = dict(facecolor=\"white\", alpha=0.3)\n",
    "\n",
    "with plt.rc_context(rc={\"font.size\": 12}):\n",
    "    fg, ax = plt.subplots(figsize=(6, 4.5))\n",
    "    ax.plot(-ddg_change, ddg_exp, \"r.\", alpha=0.5)\n",
    "    ax.set_xlabel(\"Difference in network outputs\")\n",
    "    ax.set_ylabel(\"ΔΔG (kcal / mol)\")\n",
    "    ax.set_xlim(-8, 16)\n",
    "    ax.set_ylim(-8, 16)\n",
    "    ax.text(0.04, 0.95, textstr, transform=ax.transAxes, fontsize=\"medium\", verticalalignment=\"top\", bbox=props)\n",
    "fg.tight_layout()\n",
    "# fg.savefig(NOTEBOOK_PATH.joinpath(\"protherm-all.svg\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# table = pa.Table.from_pandas(results_df)\n",
    "# pq.write_table(table, NOTEBOOK_PATH.joinpath(\"results.parquet\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [
    "1fXJz1l9HvhW",
    "u9q4iTifHvhl"
   ],
   "name": "Copy of 2019-03-30-sudoku-4xEdgeConv-09862+.ipynb",
   "provenance": [],
   "toc_visible": true,
   "version": "0.3.2"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}