notebooks/05_select_best_model.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Summary\n",
    "\n",
    "In this notebook, we select the best model based on validation data.\n",
    "\n",
    "----"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pathlib import Path"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "NOTEBOOK_NAME = \"05_select_best_model\"\n",
    "NOTEBOOK_PATH = Path(NOTEBOOK_NAME).resolve()\n",
    "NOTEBOOK_PATH.mkdir(exist_ok=True)\n",
    "NOTEBOOK_PATH"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "UNIQUE_ID = \"191f05de\"  # No attention\n",
    "# UNIQUE_ID = \"0007604c\"  # 5-layer graph-conv with attention, batch_size=1\n",
    "# UNIQUE_ID = \"91fc9ab9\"  # 4-layer graph-conv with attention, batch_size=4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATA_DIR = NOTEBOOK_PATH.parent.joinpath(\"protein_train\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "state_files = sorted(DATA_DIR.joinpath(UNIQUE_ID).glob(\"*.state\"), key=lambda s: int(s.stem.split(\"-\")[2].strip(\"d\")))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Workflow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "avg_accuracies = []\n",
    "\n",
    "for state_file_idx, state_file in enumerate(state_files):\n",
    "    net = Net(\n",
    "        x_input_size=num_features + 1, adj_input_size=adj_input_size, hidden_size=hidden_size, output_size=num_features\n",
    "    )\n",
    "    net.load_state_dict(torch.load(state_file))\n",
    "    net.eval()\n",
    "    net = net.to(device)\n",
    "\n",
    "    results = []\n",
    "    for i, data in enumerate(\n",
    "        tqdm.tqdm_notebook(\n",
    "            DataLoader(datasets[\"protein_valid\"], shuffle=False, num_workers=1, batch_size=1, drop_last=False),\n",
    "            leave=False,\n",
    "            desc=f\"{state_file_idx}\",\n",
    "        )\n",
    "    ):\n",
    "        data = data.to(device)\n",
    "        data.y = data.x\n",
    "        x_in = torch.ones_like(data.y) * 20.0\n",
    "        is_missing = torch.ones(data.y.size(0), dtype=torch.bool)\n",
    "        output = net(x_in, data.edge_index, data.edge_attr)\n",
    "        output = torch.softmax(output, dim=1)\n",
    "        _, predicted = output.max(dim=1)\n",
    "        num_correct = float((predicted[is_missing] == data.y[is_missing]).sum())\n",
    "        num_total = float(is_missing.sum())\n",
    "        results.append(\n",
    "            {\"fraction_correct\": num_correct / num_total, \"num_correct\": num_correct, \"num_total\": num_total}\n",
    "        )\n",
    "\n",
    "    oneshot_results_df = pd.DataFrame(results)\n",
    "\n",
    "    datapoint = int(state_file.stem.split(\"-\")[2].strip(\"d\"))\n",
    "    avg_accuracies.append((state_file_idx, datapoint, oneshot_results_df[\"fraction_correct\"].mean()))\n",
    "    print(avg_accuracies[-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "_, datapoints, accuracies = np.array(avg_accuracies).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fg, ax = plt.subplots()\n",
    "# ax.axhline(0.24, color='k', linestyle='--')\n",
    "ax.plot(datapoints, accuracies, label=\"valid\")\n",
    "# ax.plot(valid_datapoints, valid_accuracies, label=\"valid\")\n",
    "# ax.plot(test_datapoints, test_accuracies, label=\"test\")\n",
    "ax.legend()\n",
    "ax.set_xlabel(\"Number of training data points\")\n",
    "ax.set_ylabel(\"Average test accuracy\\nwith no starting residues\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_model_index = np.argmax(accuracies)\n",
    "\n",
    "best_model_index"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}