sspred_1.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Data from http://www.compbio.dundee.ac.uk/jpred4/about_RETR_JNetv231_details.shtml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import itertools\n",
    "import pandas as pd\n",
    "import torch\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_ids = np.loadtxt('train_names', dtype='str')\n",
    "test_ids = np.loadtxt('test_names', dtype='str')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_items={}\n",
    "for i in train_ids:\n",
    "    with open('data/training/'+i+\".fasta\") as input:\n",
    "        seq = ''\n",
    "        lines = input.readlines()\n",
    "        for line in lines:\n",
    "            if line[0] != '>':\n",
    "                seq += line.strip()\n",
    "    with open('data/training/'+i+\".dssp\") as input:\n",
    "        ss = ''\n",
    "        lines = input.readlines()\n",
    "        for line in lines:\n",
    "            if line[0] != '>':\n",
    "                ss += line.strip()\n",
    "            \n",
    "    training_items[i] = {'seq':seq, 'ss':ss}\n",
    "    \n",
    "    \n",
    "test_items={}\n",
    "for i in test_ids:\n",
    "    with open('data/blind/'+i+\".fasta\") as input:\n",
    "        seq = ''\n",
    "        lines = input.readlines()\n",
    "        for line in lines:\n",
    "            if line[0] != '>':\n",
    "                seq += line.strip()\n",
    "    with open('data/blind/'+i+\".dssp\") as input:\n",
    "        ss = ''\n",
    "        lines = input.readlines()\n",
    "        for line in lines:\n",
    "            if line[0] != '>':\n",
    "                ss += line.strip()\n",
    "            \n",
    "    test_items[i] = {'seq':seq, 'ss':ss}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "aa_id_dict = {'A': 0, 'C': 1, 'D': 2, 'E': 3, 'F': 4, 'G': 5, 'H': 6, 'I': 7,\n",
    "              'K': 8, 'L': 9, 'M': 10, 'N': 11, 'P': 12, 'Q': 13, 'R': 14, \n",
    "              'S': 15, 'T': 16, 'V': 17, 'W': 18, 'Y': 19}\n",
    "\n",
    "pos_aa_dict = {j:i for i,j in aa_id_dict.items()}\n",
    "\n",
    "ss_id_dict = {'H':0, 'E':1, '-':2}\n",
    "\n",
    "pos_ss_dict = {j:i for i,j in ss_id_dict.items()}\n",
    "\n",
    "def aa_to_onehot(aa_str, aa_to_nr=aa_id_dict, mask=None):\n",
    "    \"\"\"\n",
    "    Onehot encode an amino acid string using a letter to number dictionary.\n",
    "    The mask (from proteinnet files) is used to remove residues missing atoms from the primary sequence.\n",
    "    \"\"\"\n",
    "    if mask!=None:\n",
    "        mask_ind = np.asarray([x=='+' for x in mask])*1\n",
    "        mask_ind = np.nonzero(mask_ind)\n",
    "        aa_str = \"\".join([aa_str[x] for x in mask_ind[0]]) # the mask indices are a list in a list\n",
    "    init_array = np.zeros( (len(aa_to_nr.keys()), len(aa_str)) )\n",
    "    for i,j in enumerate(aa_str):\n",
    "        init_array[aa_to_nr[j], i] = 1\n",
    "    return init_array\n",
    "\n",
    "def label_to_index(ss, id_dict):\n",
    "    labels = np.array([id_dict[i] for i in ss])\n",
    "    return(labels)\n",
    "\n",
    "def onehot_to_str(onehot_arr, map_dict=pos_aa_dict):\n",
    "    '''Helper function to recover aa sequence from onehot encoding\n",
    "        input must be aa*N numpy array'''\n",
    "    aas = []\n",
    "    N = onehot_arr.shape[1]\n",
    "    for i in range(N):\n",
    "        pos = np.where(onehot_arr[:, i]>0)[0]\n",
    "        aas.append(map_dict[int(pos)])\n",
    "    return \"\".join(aas)\n",
    "\n",
    "def filter_proteins(prot_id, seq, allowed_symbols):\n",
    "    allowed = True\n",
    "    for i in seq:\n",
    "        if i not in allowed_symbols:\n",
    "            allowed = False\n",
    "    return allowed\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1, 20, 401)"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "training_items[train_ids_filt[0]]['seq_1h'].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_inds_filt = np.array([filter_proteins(i, training_items[i]['seq'], aa_id_dict.keys()) for i in train_ids])\n",
    "test_inds_filt = np.array([filter_proteins(i, test_items[i]['seq'], aa_id_dict.keys()) for i in test_ids])\n",
    "\n",
    "train_ids_filt = train_ids[train_inds_filt]\n",
    "test_ids_filt = test_ids[test_inds_filt]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1345,)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_ids_filt.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in train_ids_filt:\n",
    "    training_items[i]['seq_1h'] = aa_to_onehot(training_items[i]['seq'], aa_id_dict)[np.newaxis, :, :]\n",
    "    training_items[i]['ss_1h'] = aa_to_onehot(training_items[i]['ss'], ss_id_dict)[np.newaxis, :, :]\n",
    "    \n",
    "for i in test_ids_filt:\n",
    "    test_items[i]['seq_1h'] = aa_to_onehot(test_items[i]['seq'], aa_id_dict)[np.newaxis, :, :]\n",
    "    test_items[i]['ss_1h'] = aa_to_onehot(test_items[i]['ss'], ss_id_dict)[np.newaxis, :, :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(678)\n",
    "inds_perm = np.random.permutation(len(train_ids_filt))\n",
    "val_prots = train_ids_filt[inds_perm[0:int(np.floor(len(inds_perm)*0.2))]]\n",
    "train_prots = train_ids_filt[inds_perm[int(np.floor(len(inds_perm)*0.2)):]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.utils.data import Dataset\n",
    "class proteindataset(torch.utils.data.Dataset):\n",
    "    def __init__(self, seqs, ss):\n",
    "        self.sequences = seqs\n",
    "        self.ss = ss\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.sequences)\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        return [self.sequences[idx], self.ss[idx]]\n",
    "\n",
    "def protein_collate(batch):\n",
    "    seqs = [item[0] for item in batch]\n",
    "    ss = [item[1] for item in batch]\n",
    "    max_len = max([x.shape[2] for x in seqs])\n",
    "    for i in range(len(batch)):\n",
    "        curr_len = seqs[i].shape[2]\n",
    "        seq_padded = np.pad(seqs[i], ((0,0 ), (0,0), (0,max_len-curr_len)), constant_values = 0)\n",
    "        ss_padded = np.pad(ss[i], ((0,0 ), (0,0), (0,max_len-curr_len)), constant_values = 0)\n",
    "        seqs[i] = torch.tensor(seq_padded).float()\n",
    "        ss[i] = torch.tensor(ss_padded).float()\n",
    "    seq_tensor = torch.cat(seqs, 0)\n",
    "    ss_tensor = torch.cat(ss, 0)\n",
    "    return [seq_tensor, ss_tensor]\n",
    "\n",
    "def protein_collate2(batch):\n",
    "    seqs = [item[0] for item in batch]\n",
    "    ss = [item[1] for item in batch]\n",
    "    max_len = max([x.shape[2] for x in seqs])\n",
    "    for i in range(len(batch)):\n",
    "        curr_len = seqs[i].shape[2]\n",
    "        seq_padded = np.pad(seqs[i], ((0,0 ), (0,0), (0,max_len-curr_len)), constant_values = 0)\n",
    "        ss_padded = np.pad(ss[i], ((0,0 ), (0,max_len-curr_len)), constant_values = 0)\n",
    "        seqs[i] = torch.tensor(seq_padded).float()\n",
    "        ss[i] = torch.tensor(ss_padded).float()\n",
    "    seq_tensor = torch.cat(seqs, 0)\n",
    "    ss_tensor = torch.cat(ss, 0)\n",
    "    return [seq_tensor, ss_tensor]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_seqs = [training_items[i]['seq_1h'] for i in train_prots]\n",
    "train_ss = [training_items[i]['ss_1h'] for i in train_prots]\n",
    "\n",
    "val_seqs = [training_items[i]['seq_1h'] for i in val_prots]\n",
    "val_ss = [training_items[i]['ss_1h'] for i in val_prots]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dataset = proteindataset(train_seqs, train_ss)\n",
    "val_dataset = proteindataset(val_seqs, val_ss)\n",
    "test_dataset = proteindataset([test_items[i]['seq_1h'] for i in test_items.keys()], [test_items[i]['ss_1h'] for i in test_items.keys()])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1, 3, 89)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_ss[0].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=4,\n",
    "                                          shuffle=True, num_workers=2, collate_fn=protein_collate)\n",
    "valloader = torch.utils.data.DataLoader(val_dataset, batch_size=1,\n",
    "                                         shuffle=False, num_workers=4, collate_fn=protein_collate)\n",
    "testloader = torch.utils.data.DataLoader(test_dataset, batch_size=1,\n",
    "                                         shuffle=False, num_workers=4, collate_fn=protein_collate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "\n",
    "kd2 = 3\n",
    "pad2 = int((kd2-1)/2)\n",
    "kd3 = 5\n",
    "pad3 = int((kd3-1)/2)\n",
    "kd4 = 7\n",
    "pad4 = int((kd4-1)/2)\n",
    "\n",
    "class Net(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(Net, self).__init__()\n",
    "        self.pool = nn.MaxPool1d(2)\n",
    "        \n",
    "        self.conv0 = nn.Conv1d(20, 32, kernel_size=1)\n",
    "        self.conv0_bn = torch.nn.BatchNorm1d(32)\n",
    "        self.conv1 = nn.Conv1d(32, 32, kernel_size=kd2, padding=pad2) # down\n",
    "        self.conv1_bn = torch.nn.BatchNorm1d(32)\n",
    "        self.conv2 = nn.Conv1d(32, 64, kernel_size=kd3, padding=pad3) # down \n",
    "        self.conv2_bn = torch.nn.BatchNorm1d(64)\n",
    "        self.conv3 = nn.Conv1d(64, 64, kernel_size=kd4, padding=pad4) # down\n",
    "        self.conv3_bn = torch.nn.BatchNorm1d(64)\n",
    "        self.conv4 = nn.Conv1d(64, 128, kernel_size=1)\n",
    "        self.conv4_bn = torch.nn.BatchNorm1d(128)\n",
    "        \n",
    "        self.deconv1 = nn.ConvTranspose1d(in_channels=128, out_channels=64, kernel_size=kd4, padding=pad4) # up\n",
    "        self.deconv1_bn = torch.nn.BatchNorm1d(64)\n",
    "        self.conv5 = nn.Conv1d(64, 64, 1)\n",
    "        self.conv5_bn = torch.nn.BatchNorm1d(64)\n",
    "        \n",
    "        self.deconv2 = nn.ConvTranspose1d(in_channels=128, out_channels=64, kernel_size=kd3, padding=pad3) # up\n",
    "        self.deconv2_bn = torch.nn.BatchNorm1d(64)\n",
    "        self.conv6 = nn.Conv1d(64, 64, 1)\n",
    "        self.conv6_bn = torch.nn.BatchNorm1d(64)\n",
    "        \n",
    "        self.deconv3 = nn.ConvTranspose1d(in_channels=96, out_channels=32, kernel_size=kd2, padding=pad2) # up\n",
    "        self.deconv3_bn = torch.nn.BatchNorm1d(32)\n",
    "        self.conv7 = nn.Conv1d(32, 16, (1))\n",
    "        self.conv7_bn = torch.nn.BatchNorm1d(16)\n",
    "        self.conv8 = nn.Conv1d(16, 3, 1)\n",
    "        self.conv8_bn = torch.nn.BatchNorm1d(3)\n",
    "        self.conv9 = nn.Conv1d(3, 3, 1)\n",
    "\n",
    "    def forward(self, x):\n",
    "        conv0_out = F.relu(self.conv0_bn(self.conv0(x)))\n",
    "        conv1_out = F.relu(self.conv1_bn(self.conv1(conv0_out)))\n",
    "        conv2_out = F.relu(self.conv2_bn(self.conv2(conv1_out)))\n",
    "        conv3_out = F.relu(self.conv3_bn(self.conv3(conv2_out)))\n",
    "        conv4_out = F.relu(self.conv4_bn(self.conv4(conv3_out)))\n",
    "\n",
    "        deconv1_out = F.relu(self.deconv1_bn(self.deconv1(conv4_out)))\n",
    "        conv5_out = F.relu(self.conv5_bn(self.conv5(deconv1_out)))\n",
    "        \n",
    "        deconv2_input = torch.cat((conv2_out, deconv1_out), 1)  \n",
    "        deconv2_out = F.relu(self.deconv2_bn(self.deconv2(deconv2_input)))\n",
    "        conv6_out = F.relu(self.conv6_bn(self.conv6(deconv2_out)))\n",
    "        \n",
    "        deconv3_input = torch.cat((conv1_out, deconv2_out), 1)\n",
    "        deconv3_out = F.relu(self.deconv3_bn(self.deconv3(deconv3_input)))\n",
    "        conv7_out = F.relu(self.conv7_bn(self.conv7(deconv3_out)))\n",
    "        conv8_out = F.relu(self.conv8(conv7_out))\n",
    "        conv9_out = self.conv9(conv8_out)\n",
    "        return conv9_out\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 0, iteration: 88] training loss: 0.7723159173901162, validation_loss: 0.7057820523095397\n",
      "epoch: 0, iteration: 177] training loss: 0.7430565015653546, validation_loss: 0.6895521125828905\n",
      "new best validation loss, saving..\n",
      "epoch: 0, iteration: 266] training loss: 0.7310940040631241, validation_loss: 0.6816603235595733\n",
      "new best validation loss, saving..\n",
      "epoch: 1, iteration: 88] training loss: 0.72035234028034, validation_loss: 0.6631677858669963\n",
      "new best validation loss, saving..\n",
      "epoch: 1, iteration: 177] training loss: 0.7194610117526536, validation_loss: 0.656789412618127\n",
      "new best validation loss, saving..\n",
      "epoch: 1, iteration: 266] training loss: 0.7202514517173338, validation_loss: 0.6703418189708185\n",
      "epoch: 2, iteration: 88] training loss: 0.7091167944200923, validation_loss: 0.6482123733674724\n",
      "new best validation loss, saving..\n",
      "epoch: 2, iteration: 177] training loss: 0.7037920750928729, validation_loss: 0.6442042827384623\n",
      "new best validation loss, saving..\n",
      "epoch: 2, iteration: 266] training loss: 0.7047164252634799, validation_loss: 0.6444184262734809\n",
      "epoch: 3, iteration: 88] training loss: 0.6962266457214784, validation_loss: 0.641398904824346\n",
      "new best validation loss, saving..\n",
      "epoch: 3, iteration: 177] training loss: 0.701837139852931, validation_loss: 0.6526511013507842\n",
      "epoch: 3, iteration: 266] training loss: 0.6899585891305731, validation_loss: 0.6386492004181817\n",
      "new best validation loss, saving..\n",
      "epoch: 4, iteration: 88] training loss: 0.6822990313004912, validation_loss: 0.6357350226450142\n",
      "new best validation loss, saving..\n",
      "epoch: 4, iteration: 177] training loss: 0.6947032836046112, validation_loss: 0.6521045800478484\n",
      "epoch: 4, iteration: 266] training loss: 0.6873139665367898, validation_loss: 0.6402358674205373\n",
      "epoch: 5, iteration: 88] training loss: 0.6803762443949667, validation_loss: 0.641265894732954\n",
      "epoch: 5, iteration: 177] training loss: 0.6841703318478016, validation_loss: 0.6381661411111683\n",
      "epoch: 5, iteration: 266] training loss: 0.6861875043826157, validation_loss: 0.6338024590316761\n",
      "new best validation loss, saving..\n",
      "epoch: 6, iteration: 88] training loss: 0.6729335014739734, validation_loss: 0.663560748875806\n",
      "epoch: 6, iteration: 177] training loss: 0.6849739605121399, validation_loss: 0.6349666372329326\n",
      "epoch: 6, iteration: 266] training loss: 0.6841510306583362, validation_loss: 0.6365910677661689\n",
      "epoch: 7, iteration: 88] training loss: 0.6733093295204505, validation_loss: 0.6361880249250335\n",
      "epoch: 7, iteration: 177] training loss: 0.6796296895220039, validation_loss: 0.6305502026054489\n",
      "new best validation loss, saving..\n",
      "epoch: 7, iteration: 266] training loss: 0.6686140446180708, validation_loss: 0.6370542295360211\n",
      "epoch: 8, iteration: 88] training loss: 0.6718968742349175, validation_loss: 0.6448175069567882\n",
      "epoch: 8, iteration: 177] training loss: 0.672168697199125, validation_loss: 0.6384177727548603\n",
      "epoch: 8, iteration: 266] training loss: 0.6786246755149927, validation_loss: 0.6343770296378647\n",
      "epoch: 9, iteration: 88] training loss: 0.6623082368561392, validation_loss: 0.6399514986458322\n",
      "Finished Training\n"
     ]
    }
   ],
   "source": [
    "net = Net()\n",
    "\n",
    "#criterion = nn.CrossEntropyLoss()\n",
    "class BinaryDiceLoss(nn.Module):\n",
    "    \"\"\"Dice loss of binary class\n",
    "    Args:\n",
    "        smooth: A float number to smooth loss, and avoid NaN error, default: 1\n",
    "        p: Denominator value: \\sum{x^p} + \\sum{y^p}, default: 2\n",
    "        predict: A tensor of shape [N, *]\n",
    "        target: A tensor of shape same with predict\n",
    "        reduction: Reduction method to apply, return mean over batch if 'mean',\n",
    "            return sum if 'sum', return a tensor of shape [N,] if 'none'\n",
    "    Returns:\n",
    "        Loss tensor according to arg reduction\n",
    "    Raise:\n",
    "        Exception if unexpected reduction\n",
    "    \"\"\"\n",
    "    def __init__(self, smooth=1, p=2, reduction='mean'):\n",
    "        super(BinaryDiceLoss, self).__init__()\n",
    "        self.smooth = smooth\n",
    "        self.p = p\n",
    "        self.reduction = reduction\n",
    "\n",
    "    def forward(self, predict, target):\n",
    "        assert predict.shape[0] == target.shape[0], \"predict & target batch size don't match\"\n",
    "        predict = predict.contiguous().view(predict.shape[0], -1)\n",
    "        target = target.contiguous().view(target.shape[0], -1)\n",
    "\n",
    "        num = torch.sum(torch.mul(predict, target), dim=1) + self.smooth\n",
    "        den = torch.sum(predict.pow(self.p) + target.pow(self.p), dim=1) + self.smooth\n",
    "\n",
    "        loss = 1 - num / den\n",
    "\n",
    "        if self.reduction == 'mean':\n",
    "            return loss.mean()\n",
    "        elif self.reduction == 'sum':\n",
    "            return loss.sum()\n",
    "        elif self.reduction == 'none':\n",
    "            return loss\n",
    "        else:\n",
    "            raise Exception('Unexpected reduction {}'.format(self.reduction))\n",
    "\n",
    "class DiceLoss(nn.Module):\n",
    "    \"\"\"Dice loss, need one hot encode input\n",
    "    Args:\n",
    "        weight: An array of shape [num_classes,]\n",
    "        ignore_index: class index to ignore\n",
    "        predict: A tensor of shape [N, C, *]\n",
    "        target: A tensor of same shape with predict\n",
    "        other args pass to BinaryDiceLoss\n",
    "    Return:\n",
    "        same as BinaryDiceLoss\n",
    "    \"\"\"\n",
    "    def __init__(self, weight=None, ignore_index=None, **kwargs):\n",
    "        super(DiceLoss, self).__init__()\n",
    "        self.kwargs = kwargs\n",
    "        self.weight = weight\n",
    "        self.ignore_index = ignore_index\n",
    "\n",
    "    def forward(self, predict, target):\n",
    "        #print(predict.shape, target.shape)\n",
    "        assert predict.shape == target.shape, 'predict & target shape do not match'\n",
    "        dice = BinaryDiceLoss(**self.kwargs)\n",
    "        total_loss = 0\n",
    "        predict = F.softmax(predict, dim=1)\n",
    "\n",
    "        for i in range(target.shape[1]):\n",
    "            if i != self.ignore_index:\n",
    "                dice_loss = dice(predict[:, i], target[:, i])\n",
    "                if self.weight is not None:\n",
    "                    assert self.weight.shape[0] == target.shape[1], \\\n",
    "                        'Expect weight shape [{}], get[{}]'.format(target.shape[1], self.weight.shape[0])\n",
    "                    dice_loss *= self.weights[i]\n",
    "                total_loss += dice_loss\n",
    "\n",
    "        return total_loss/target.shape[1]\n",
    "\n",
    "criterion = DiceLoss()\n",
    "optimizer = optim.Adam(net.parameters(), lr=1e-3)\n",
    "\n",
    "prints_per_epoch = 3\n",
    "\n",
    "verbose_k = np.floor(len(trainloader)/prints_per_epoch)\n",
    "\n",
    "\n",
    "train_losses = []\n",
    "val_losses = []\n",
    "iterations = []\n",
    "best_loss = None\n",
    "patience_val = 5\n",
    "patience_counter = patience_val\n",
    "\n",
    "for epoch in range(10):\n",
    "\n",
    "    running_loss = 0.0\n",
    "    for i, data in enumerate(trainloader):\n",
    "        sequence, true_angles = data\n",
    "        #print(sequence.shape, true_angles.shape)\n",
    "        # zero the parameter gradients\n",
    "        optimizer.zero_grad()\n",
    "\n",
    "        # forward + backward + optimize\n",
    "        predicted_angles = net(sequence)\n",
    "\n",
    "        loss = criterion(predicted_angles, true_angles)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        \n",
    "        # adding to running loss, we will output this at every verbose_k\n",
    "        running_loss += loss.item()\n",
    "        \n",
    "        if (i+1) % verbose_k == 0:\n",
    "            if patience_counter < 1:\n",
    "                break\n",
    "            train_losses.append(running_loss/verbose_k)\n",
    "            true_iter = len(trainloader)*epoch + i\n",
    "            iterations.append(true_iter)\n",
    "            net.eval()\n",
    "            validation_loss = 0\n",
    "            for j in valloader:\n",
    "                pred_k = net(j[0])\n",
    "                loss_k = criterion(pred_k, j[1]).item()\n",
    "                validation_loss += loss_k/len(val_seqs)\n",
    "            val_losses.append(validation_loss)\n",
    "            net.train()\n",
    "            print('epoch: {}, iteration: {}] training loss: {}, validation_loss: {}'.format(\n",
    "                epoch, i, running_loss/verbose_k, validation_loss))\n",
    "\n",
    "            if best_loss == None:\n",
    "                best_loss = validation_loss\n",
    "            else:\n",
    "                if validation_loss <= min(val_losses):\n",
    "                    patience_counter = patience_val\n",
    "                    print('new best validation loss, saving..')\n",
    "                    best_loss = validation_loss\n",
    "                    torch.save(net.state_dict(), 'best_fcn_parameters.pt')\n",
    "                else:\n",
    "                    patience_counter -= 1\n",
    "            \n",
    "\n",
    "            running_loss = 0.0\n",
    "            \n",
    "\n",
    "print('Finished Training')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "net.load_state_dict(torch.load('/Users/Deathvoodoo/big_folders_docs/ss_pred/best_fcn_parameters.pt'))\n",
    "\n",
    "net.eval()\n",
    "\n",
    "validation_preds = [np.apply_along_axis(np.argmax, 1, net(i[0]).detach().numpy()) for i in valloader]\n",
    "#validation_reals = [np.apply_along_axis(np.argmax, 1, i[1].detach().numpy()) for i in valloader]\n",
    "validation_reals = [np.apply_along_axis(np.argmax, 1, i) for i in val_ss]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD8CAYAAACb4nSYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3X90VeW95/H31xgE0RKq6JIEJXZQfoRIIHLt5SJaq6KtP1fVWJ2qM5WpldprVxlj71VZ3jtLW7Va5tJZi3rtsraVshil9KqlVLFql3YIPxUQRaCS0NFICYWCU9Dv/HF28BDyY+/k7Jy9z/681srKOc959snz5Jzz3fs8z7O/29wdERHJhiOK3QAREek/CvoiIhmioC8ikiEK+iIiGaKgLyKSIQr6IiIZoqAvIpIhCvoiIhmioC8ikiFHFrsBHR1//PE+cuTIYjdDRCRVVqxY8YG7D+upXuKC/siRI2lqaip2M0REUsXM/himnoZ3REQyREFfRCRDFPRFRDIkcWP6ItL/9u/fT3NzMx9++GGxmyI9GDhwIFVVVZSXl/dqewV9EaG5uZljjz2WkSNHYmbFbo50wd3ZsWMHzc3NVFdX9+o5NLwjInz44Yccd9xxCvgJZ2Ycd9xxffpGpqAvIgAK+CnR19dJQV9EJEMU9EWk6Nra2vjhD3/Yq20vvvhi2traQtefPXs2Dz74YK/+VilQ0BeRousu6B84cKDbbZ999lkqKiriaFZJUtAXkcgWrWphyv0vUN34DFPuf4FFq1r69HyNjY288847TJgwgVmzZvHiiy8ydepULr30UsaOHQvA5ZdfzqRJkxg3bhzz5s07uO3IkSP54IMP2Lp1K2PGjOHmm29m3LhxXHDBBezbt6/bv7t69WrOOussamtrueKKK9i5cycAc+bMYezYsdTW1tLQ0ADA7373OyZMmMCECROoq6tj9+7dfepz0bh7on4mTZrkItK/1q9fH7ru0yubffQ/P+en3PEfB39G//Nz/vTK5l7//S1btvi4ceMO3l+2bJkfffTRvnnz5oNlO3bscHf3vXv3+rhx4/yDDz5wd/dTTjnFW1tbfcuWLV5WVuarVq1yd/errrrKn3jiicP+1j333OMPPPCAu7uPHz/eX3zxRXd3v+uuu/yb3/ymu7ufdNJJ/uGHH7q7+86dO93d/Ytf/KK/8sor7u6+e/du379/f6/721edvV5Ak4eIsaGO9M1supltNLNNZtbYyeMPm9nq4OctM2vLe+x7ZrbOzDaY2RzTEgGRVHtgyUb27f/okLJ9+z/igSUbC/p3Jk+efMha9Dlz5nDGGWdw1llnsW3bNt5+++3DtqmurmbChAkATJo0ia1bt3b5/Lt27aKtrY1p06YBcMMNN/DSSy8BUFtby3XXXcdPf/pTjjwydzrTlClT+Na3vsWcOXNoa2s7WJ42PQZ9MysD5gIXAWOBa81sbH4dd7/d3Se4+wTgfwJPBdv+PTAFqAVqgDOBaQXtgYj0q+1tnQ+ZdFXeW4MHDz54+8UXX+S3v/0tr776KmvWrKGurq7TtepHHXXUwdtlZWU9zgd05ZlnnuHWW29l5cqVnHnmmRw4cIDGxkYeffRR9u3bx5QpU3jzzTd79dzFFuZIfzKwyd03u/vfgPnAZd3UvxZ4MrjtwEBgAHAUUA681/vmikixDa8YFKk8jGOPPbbbMfJdu3YxdOhQjj76aN58801ee+21Xv+tdkOGDGHo0KG8/PLLADzxxBNMmzaNjz/+mG3btnHuuefy3e9+l127drFnzx7eeecdxo8fzx133MGZZ55Z0kG/EtiWd785KDuMmZ0CVAMvALj7q8Ay4E/BzxJ339CXBotIcc268HQGlZcdUjaovIxZF57e6+c87rjjmDJlCjU1NcyaNeuwx6dPn86BAwcYM2YMjY2NnHXWWb3+W/kef/xxZs2aRW1tLatXr+buu+/mo48+4vrrr2f8+PHU1dVx2223UVFRwSOPPEJNTQ21tbWUl5dz0UUXFaQN/c1y4//dVDD7EjDd3b8a3P/PwN+5+8xO6t4BVLn7N4L7/wn4AXBNUGUp8N/d/eUO280AZgCcfPLJk/74x1DXAhCRAtmwYQNjxowJXX/RqhYeWLKR7W37GF4xiFkXns7ldZ0eC0oMOnu9zGyFu9f3tG2YmYgWYETe/aqgrDMNwK15968AXnP3PUGjngM+CxwS9N19HjAPoL6+vvu9kIgU3eV1lQryKRVmeGc5MMrMqs1sALnAvrhjJTMbDQwFXs0rfheYZmZHmlk5uUlcDe+IiBRJj0Hf3Q8AM4El5AL2AndfZ2b3mtmleVUbgPl+6HjRQuAd4HVgDbDG3X9VsNaLiEgkoRaauvuzwLMdyu7ucH92J9t9BPy3PrRPREQKSGkYREQyREFfRCRDSjLoFzoZlIgkzzHHHAPA9u3b+dKXvtRpnXPOOYempqZun+eRRx5h7969B+9HTdXclaSmcC65oL9oVQt3PvU6LW37cKClbR93PvW6Ar9IiRo+fDgLFy7s9fYdg36pp2ouuaDfX8mgRDJt7QJ4uAZmV+R+r13Qp6drbGxk7ty5B++3HyXv2bOH8847j4kTJzJ+/Hh++ctfHrbt1q1bqampAWDfvn00NDQwZswYrrjiikNSK99yyy3U19czbtw47rnnHiCXxG379u2ce+65nHvuucAnqZoBvv/971NTU0NNTQ2PPPLIwb+X6hTOYVJx9udPX1Mrj8xL95r/M/KO/+jT84qUsiiplX3NL9z/9UT3ez71yc+/npgr76WVK1f62WefffD+mDFj/N133/X9+/f7rl273N29tbXVP/OZz/jHH3/s7u6DBw9290PTMj/00EN+00035Zq5Zo2XlZX58uXL3f2T1MwHDhzwadOm+Zo1a9z9k9TM7drvNzU1eU1Nje/Zs8d3797tY8eO9ZUrVyYihXPsqZXTJI5kUCKS5/l7YX+HI9v9+3LlvVRXV8f777/P9u3bWbNmDUOHDmXEiBG4O9/5zneora3l85//PC0tLbz3Xtc5G1966SWuv/56IJceuba29uBjCxYsYOLEidTV1bFu3TrWr1/fbZteeeUVrrjiCgYPHswxxxzDlVdeeTA5W5pTOJdc0I8jGZSI5NnVHK08pKuuuoqFCxfyi1/8gmuuyaXr+tnPfkZraysrVqxg9erVnHjiiZ2mVO7Jli1bePDBB3n++edZu3YtX/jCF3r1PO3SnMK55IL+5XWV3HfleCorBmFAZcUg7rtyvPKEiBTKkKpo5SFdc801zJ8/n4ULF3LVVVcBuaPkE044gfLycpYtW0ZPyRjPPvtsfv7znwPwxhtvsHbtWgD+8pe/MHjwYIYMGcJ7773Hc889d3CbrtI6T506lUWLFrF3717++te/8vTTTzN16tTI/UpaCud0XvqlB0oGJRKj8+6GX9126BBP+aBceR+MGzeO3bt3U1lZyUknnQTAddddxyWXXML48eOpr69n9OjR3T7HLbfcwk033cSYMWMYM2YMkyZNAuCMM86grq6O0aNHM2LECKZMmXJwmxkzZjB9+nSGDx/OsmXLDpZPnDiRG2+8kcmTJwPw1a9+lbq6um6Hcrry+OOP87WvfY29e/dy6qmn8uMf//hgCuddu3bh7gdTON91110sW7aMI444gnHjxhU8hXOPqZX7W319vfe0rlZECitqamXWLsiN4e9qzh3hn3c31F4dXwPlEHGnVhYROVTt1QryKVVyY/oiItI1BX0RAXLn7Ejy9fV1UtAXEQYOHMiOHTsU+BPO3dmxYwcDBw7s9XNoTF9EqKqqorm5mdbW1mI3RXowcOBAqqp6vzw2VNA3s+nkLnBeBjzq7vd3ePxh4Nzg7tHACe5eETx2MvAouevsOnCxu2/tdYtFpODKy8uprq4udjOkH/QY9M2sDJgLnA80A8vNbLG7HzyH2d1vz6v/DaAu7yl+AvwPd19qZscAHxeq8SIiEk2YMf3JwCZ33+zufwPmA5d1U/9a4EkAMxsLHOnuSwHcfY+77+1mWxERiVGYoF8JbMu73xyUHcbMTgGqgReCotOANjN7ysxWmdkDwTcHEREpgkKv3mkAFnruguiQGz6aCnwbOBM4Fbix40ZmNsPMmsysSRNJIiLxCRP0W8hNwrarCso600AwtBNoBlYHQ0MHgEXAxI4bufs8d6939/phw4aFa7mIiEQWJugvB0aZWbWZDSAX2Bd3rGRmo4GhwKsdtq0ws/ZI/jmg+yTWIiISmx6DfnCEPhNYAmwAFrj7OjO718wuzavaAMz3vLM7gmGebwPPm9nrgAE/KmQHREQkPGXZFBEpAWGzbCoNg4hIhijoi4hkiIK+iEiGKOiLiGSIgr6ISIYo6IuIZIiCvohIhijoi4hkiIK+iEiGKOiLiGSIgr6ISIYo6IuIZIiCvohIhvR4YfRSt2hVCw8s2cj2tn0MrxjErAtP5/K6Tq8GKSKSepkO+otWtXDnU6+zb3/u6o4tbfu486nXART4RaQkZXp454ElGw8G/Hb79n/EA0s2FqlFIiLxChX0zWy6mW00s01m1tjJ4w+b2erg5y0za+vw+KfMrNnM/q1QDS+E7W37IpWLiKRdj8M7ZlYGzAXOJ3eh8+VmttjdD17r1t1vz6v/DaCuw9P8C/BSQVpcQMMrBtHSSYAfXjGoCK0REYlfmCP9ycAmd9/s7n8D5gOXdVP/WuDJ9jtmNgk4EfhNXxoah1kXns6g8rJDygaVlzHrwtOL1CIRkXiFCfqVwLa8+81B2WHM7BSgGnghuH8E8BC5i6MnzuV1ldx35XgqKwZhQGXFIO67crwmcUWkZBV69U4DsNDd22dHvw486+7NZtblRmY2A5gBcPLJJxe4Sd27vK5SQV5EMiNM0G8BRuTdrwrKOtMA3Jp3/7PAVDP7OnAMMMDM9rj7IZPB7j4PmAdQX1/vIdsuIiIRhQn6y4FRZlZNLtg3AF/uWMnMRgNDgVfby9z9urzHbwTqOwZ8ERHpPz2O6bv7AWAmsATYACxw93Vmdq+ZXZpXtQGY7+46UhcRSShLWoyur6/3pqamYjdDRCRVzGyFu9f3VC/TZ+SKiGSNgr6ISIYo6IuIZEims2xGpTTMIpJ2CvohKQ2ziJQCDe+EpDTMIlIKFPRDUhpmESkFCvohdZVuWWmYRSRNFPRDUhpmESkFmsgNqX2yVqt3RCTNFPQjUBpmEUk7De+IiGSIgr6ISIYo6IuIZIiCvohIhijoi4hkSKigb2bTzWyjmW0ys8Mud2hmD5vZ6uDnLTNrC8onmNmrZrbOzNaa2TWF7oCIiITX45JNMysD5gLnA83AcjNb7O7r2+u4++159b8B1AV39wJfcfe3zWw4sMLMlrh7WyE7kURxZeRUpk8R6Ysw6/QnA5vcfTOAmc0HLgPWd1H/WuAeAHd/q73Q3beb2fvAMKCkg37UjJxhA7kyfYpIX4UZ3qkEtuXdbw7KDmNmpwDVwAudPDYZGAC8E72Z6RIlI2d7IG9p24fzSSBftKqlT88rItKZQk/kNgAL3f2QyGRmJwFPADe5+8cdNzKzGWbWZGZNra2tBW5S/4uSkTNKIFemTxHpqzBBvwUYkXe/KijrTAPwZH6BmX0KeAb4J3d/rbON3H2eu9e7e/2wYcNCNCnZomTkjBLIlelTRPoqTNBfDowys2ozG0AusC/uWMnMRgNDgVfzygYATwM/cfeFhWly8kXJyBklkCvTp4j0VY9B390PADOBJcAGYIG7rzOze83s0ryqDcB8d/e8squBs4Eb85Z0Tihg+xPp8rpK7rtyPJUVgzCgsmIQ9105vtPJ1iiBPMrzioh0xg6N0cVXX1/vTU1NxW5Gv9IyTBHpKzNb4e71PdVTauUEUMpmEekvSsMgIpIhCvoiIhmioC8ikiEK+iIiGaKgLyKSIVq9U8K0FFREOlLQL1HKyCkindHwTolSRk4R6YyCfolSRk4R6UxpBv21C+DhGphdkfu9dkGxW9TvlJFTRDpTekF/7QL41W2waxvgud+/ui1zgT/OjJyLVrUw5f4XqG58hin3v9DpBV9EJJlKbyL3+Xthf4chjP37cuW1VxenTUXQPlkbdvWOLtkokg2lF/R3NUcrL2FhE7lFCeTdTRAr6IskX+kN7wypilYuumSjSIaUXtA/724o7zBZWT4oVy6dSsolGzVXIBK/0gv6tVfDJXNgyAjAcr8vmZOp8fyoknDJxvYhppa2fTifDDEp8IsUVqigb2bTzWyjmW0ys8ZOHn8473KIb5lZW95jN5jZ28HPDYVsfJdqr4bb34DZbbnfCvjdSsIlG3UymUj/6HEi18zKgLnA+UAzsNzMFrv7+vY67n57Xv1vAHXB7U8D9wD1gAMrgm13FrQX0idRV/rEcaUvzRWI9I8wq3cmA5vcfTOAmc0HLgPWd1H/WnKBHuBCYKm7/znYdikwHXiyL42Wwiv2JRuHVwyiJcIcgoj0TpjhnUpgW9795qDsMGZ2ClANvBBlWzObYWZNZtbU2toapt1SYuI8mUxEPlHoidwGYKG7f9RjzTzuPs/d6929ftiwYQVukqRBXHMFInKoMMM7LcCIvPtVQVlnGoBbO2x7TodtXwzfPEm7KDn9iz3EJJIFYY70lwOjzKzazAaQC+yLO1Yys9HAUODVvOIlwAVmNtTMhgIXBGWSAVqGKZI8PR7pu/sBM5tJLliXAY+5+zozuxdocvf2HUADMN/dPW/bP5vZv5DbcQDc2z6pK6UvjSkbdLUxKXWhcu+4+7PAsx3K7u5wf3YX2z4GPNbL9kmKpW0ZppLJSRaU3hm5khhpy+mvE8QkCxT0JTZpW4aZtm8mIr2hoC+xSdsyzLR9MxHpjdLLpy+JkqZlmLMuPP2QMX1I9jcTkd5Q0F+7IHdVrV3NuZz7592tBG0pEMcqm6g5iETSKNtBv/16uu2XV2y/ni4o8CdYnKts0vTNRKQ3sj2m3931dCWxtMpGpPeyfaSv6+mmUlJW2ehELkmjbB/p63q6qZSEVTZKMSFple2gr+vpplIS1v8nZYhJ1xWWqLI9vNM+WavVO6mShFU2SRhiijtthIavSlO2gz7kAryCfOoUe5VNnFf6Chts40xoF3WHoh1EemR7eEekl+IaYooyVxDnt40ow1ea30gXBX0peXGMe8eVYiJKsI1zQjvKDiUp8xsSjoZ3otDZu6mTthO5ogTbONNGRBm+SsL8hoSnI/2w2s/e3bUN8E/O3l27oNgtk26k7Sg0ytF7nAntogxfJWEJrYQX6kjfzKYDPyB35axH3f3+TupcDcwGHFjj7l8Oyr8HfIHcDmYp8M38q2ulRndn7+poP7GSchQadqIz6tF7XBPaUVZIKVFduvQY9M2sDJgLnA80A8vNbLG7r8+rMwq4E5ji7jvN7ISg/O+BKUBtUPUVYBppvDi6zt5NpThX2YQVZYgpCctR89sS5u/G2WatCiq8MEf6k4FN7r4ZwMzmA5cB6/Pq3AzMdfedAO7+flDuwEBgAGBAOfBeYZrez4ZUBUM7nZRLYiXhKDTq0spiL0ftjTjanMbLV6ZhJxVmTL8SyI92zUFZvtOA08zs92b2WjAchLu/CiwD/hT8LHH3DR3/gJnNMLMmM2tqbW3tTT/ip7N3UykJF3JJyhBT2qRtPiYtS1cLtXrnSGAUcA5QBbxkZuOB44ExQRnAUjOb6u4v52/s7vOAeQD19fXJHO/X2bupVewj5yQMMaVR2naWcZ4sV0hhgn4LMCLvflVQlq8Z+IO77we2mNlbfLITeM3d9wCY2XPAZ4GXSSOdvSu9kIQhpjRK284yLTupMMM7y4FRZlZtZgOABmBxhzqLyAV4zOx4csM9m4F3gWlmdqSZlZObxD1seEeklCVhiCmNkpBYL4q0LF3t8Ujf3Q+Y2UxgCbklm4+5+zozuxdocvfFwWMXmNl64CNglrvvMLOFwOeA18lN6v7a3X8VV2dEkqrYQ0xplKSVTGGk5RudJW3JfH19vTc1NRW7GSIikRVz9Y6ZrXD3+p7qKQ2DSIYkYUlhEtoQlzR8o1PQF8mIJKx7T0Ib2tsRdsdTajsp5d4RyYgkrHtPQhuirKdPy9r7KBT002btAni4BmZX5H4r4ZuElIQlhUloQ5QdTxJ2UoWmoJ8myvQpfZCEJYVJaEOUHU8SdlKFpqCfJt1l+hTpQRLWvSehDVF2PEnYSRWagn6aKNOn9EESThKLsw1hr5AWZceThJ1UoWn1Tpoo06f0URKWFBY7I2eUk77SdoJYGDo5K03ax/Tzh3jKB8Elc5QTSDJtyv0vdJqnp7JiEL9v/FwRWtT/wp6cpeGdNKm9Ohfgh4wALPdbAV+kJCdc46LhnbRRpk+Rw6QtI2cx6UhfRFKvFCdc46Ij/bisXaALrkjylOj7shQnXOOioB+HjhOu7SdRQUl8wCSlSvx9mYSVSVEUK6ePhnfioJOoJIn0vkyMYub0UdCPg06ikiTS+zIxipnTJ1TQN7PpZrbRzDaZWWMXda42s/Vmts7Mfp5XfrKZ/cbMNgSPjyxM0xOsq5OldBKVFJPel4lRzCWmPQZ9MysD5gIXAWOBa81sbIc6o4A7gSnuPg74x7yHfwI84O5jgMnA+wVqe3Kdd3fupKl85YNy5SLFovdlYhQzp0+YI/3JwCZ33+zufwPmA5d1qHMzMNfddwK4+/sAwc7hSHdfGpTvcfe9BWt9UukkKkkivS8To5hLTMOs3qkE8hO+NAN/16HOaQBm9ntyF0+f7e6/DsrbzOwpoBr4LdDo7ocMZpnZDGAGwMknn9yLbiSQTqKSJNL7MhGKucS0UEs2jwRGAecAVcBLZjY+KJ8K1AHvAr8AbgT+PX9jd58HzINc7p0CtSk9SnTttIh0rVhLTMMM77QAI/LuVwVl+ZqBxe6+3923AG+R2wk0A6uDoaEDwCJgYt+bXUJ0YRQR6Udhgv5yYJSZVZvZAKABWNyhziJyR/mY2fHkhnU2B9tWmNmwoN7ngPUFaHfp0NppEelHPQb94Ah9JrAE2AAscPd1ZnavmV0aVFsC7DCz9cAyYJa77wjG7r8NPG9mrwMG/CiOjqSW1k6LSD8KNabv7s8Cz3YouzvvtgPfCn46brsUqO1bM0uYLowiIv1IZ+QWm9ZOi0g/UtAvNq2d/sTaBfBwDcyuyP0uxmR2EtogEiNl2UwCrZ1ORgbIJLRBJGY60pdkSMIqpiS0QSRmCvqSDElYxZSENojETEFfkiEJGSCT0AaRmCnol7I0TUomYRVTEtogEjNN5JaqtE1KtrepmDmIktAGkZhZ7ryq5Kivr/empqZiNyP9Hq7p4qSvEXD7G/3fHhGJlZmtcPf6nuppeKdUaVJSRDqhoF+qNCkp0rU0zXcVmIJ+qYo6KRnlQ5DhD4yUgIynM1fQL1VR0jtE+RBk/AMjJSDjJ+Fp9U4pC5veobsPQcfto9QVSaKMz3fpSF+ifQgy/oGREpDx+S4FfYn2IYj6gdH4f47+D8mR8ZPwQgV9M5tuZhvNbJOZNXZR52ozW29m68zs5x0e+5SZNZvZvxWi0VJgUT4EUeqW+vh/2EBe6v+HtMl4OvMeT84yszJyFzo/n9yFzpcD17r7+rw6o4AFwOfcfaeZneDu7+c9/gNgGPBnd5/Z3d/TyVlFsnZB+DNRw9Yt5RPEOp7xDLmdX2fBo5T/D9J7UT5zIYQ9OSvMRO5kYJO7bw6eeD5wGYde4PxmYK677wToEPAnAScCvwZ6bJAUSZSc/mHrxjn+X+APTGRRJrQ1D5JucbzXipgmJczwTiWQf5jSHJTlOw04zcx+b2avmdl0ADM7AniI3MXRu2RmM8ysycyaWltbw7deki2uCbMkDJdECeSaB0mvuN5rRVw2WqiJ3COBUcA5wLXAj8ysAvg68Ky7d3tI4+7z3L3e3euHDRtWoCZJ0cU1YZaEddZRArnmQfpHHDvLuN5rRfz2FybotwAj8u5XBWX5moHF7r7f3beQmwMYBXwWmGlmW4EHga+Y2f19brWkQ1wTZkkYLokSyKP8H5KwQ+uNYp/RHdfOMq73WhGXjYYZ018OjDKzanLBvgH4coc6i8gd4f/YzI4nN9yz2d2va69gZjcC9e7e6eofKVFxXP93SFUXE6PdDJcUekw2ahrmJMyDxCXK+HRcY9lxnTQY9b0W1nl3d74QoB+WjfZ4pO/uB4CZwBJgA7DA3deZ2b1mdmlQbQmww8zWA8uAWe6+I65GS8YlZbik9urc6pvZbbnfhdi5JenEobBH5FG+naRtuCSuHFZFXDaqfPqSTqW6bDTKUtD2+nGsYorSjtkVQGdxxHI7xN7WjSLO1zns/zjqa1dghVyyKZI8pTpcEmXYKM5lf1GGS6IMgaRxuCSOHFZFpDQMUtqSNFwSVthhozgnfaPsLOM6ozuKJJxlm5IDDB3pS2kr4oRZ7OIMMlGOyKN8O4nzOsRxLBqIIq5vMQWmoC+lrZQvdh5nkIm6s4zjjG4o/pnXUaTkAENBX0pfsY8A4xL3ODYUN+AWMVVBryThfxaCVu+IpFmajoSjStvKqyLT6h2RLCjVbzGQmonRtNHqHRFJpjSuvEoBBX0RSaaMX+EqLgr6IpJMSVh7X4I0pi8iyVXKcxZFoiN9EZEMUdAXEckQBX0RkQxR0BcRyRAFfRGRDFHQFxHJEAV9EZEMUdAXEcmQxGXZNLNW4I/dVDke+KCfmpMUWewzqN9ZksU+Q2H7fYq7D+upUuKCfk/MrClM+tBSksU+g/pd7Hb0pyz2GYrTbw3viIhkiIK+iEiGpDHozyt2A4ogi30G9TtLsthnKEK/UzemLyIivZfGI30REeml1AR9M5tuZhvNbJOZNRa7PYVmZlvN7HUzW21mTUHZp81sqZm9HfweGpSbmc0J/hdrzWxicVsfnpk9Zmbvm9kbeWWR+2lmNwT13zazG4rRl7C66PNsM2sJXu/VZnZx3mN3Bn3eaGYX5pWn6jNgZiPMbJmZrTezdWb2zaC8ZF/vbvqcnNfb3RP/A5QB7wCnAgOANcDYYrerwH3cChzfoex7QGNwuxH4bnD7YuA5wICzgD8Uu/0R+nk2MBF4o7f9BD4NbA5+Dw1uDy123yL2eTbw7U7qjg15cm8+AAACkElEQVTe30cB1cH7viyNnwHgJGBicPtY4K2gfyX7enfT58S83mk50p8MbHL3ze7+N2A+cFmR29QfLgMeD24/DlyeV/4Tz3kNqDCzk4rRwKjc/SXgzx2Ko/bzQmCpu//Z3XcCS4Hp8be+d7roc1cuA+a7+/9z9y3AJnLv/9R9Btz9T+6+Mri9G9gAVFLCr3c3fe5Kv7/eaQn6lcC2vPvNdP+PTCMHfmNmK8xsRlB2orv/Kbj9f4ETg9ul9v+I2s9S6f/MYBjjsfYhDkq0z2Y2EqgD/kBGXu8OfYaEvN5pCfpZ8A/uPhG4CLjVzM7Of9Bz3wVLfqlVVvoJ/C/gM8AE4E/AQ8VtTnzM7BjgfwP/6O5/yX+sVF/vTvqcmNc7LUG/BRiRd78qKCsZ7t4S/H4feJrc17v32odtgt/vB9VL7f8RtZ+p77+7v+fuH7n7x8CPyL3eUGJ9NrNycsHvZ+7+VFBc0q93Z31O0uudlqC/HBhlZtVmNgBoABYXuU0FY2aDzezY9tvABcAb5PrYvlLhBuCXwe3FwFeC1Q5nAbvyvi6nUdR+LgEuMLOhwdfkC4Ky1OgwB3MFudcbcn1uMLOjzKwaGAX8H1L4GTAzA/4d2ODu3897qGRf7676nKjXu9iz3RFmxS8mNxP+DvBPxW5Pgft2KrnZ+TXAuvb+AccBzwNvA78FPh2UGzA3+F+8DtQXuw8R+vokua+3+8mNU/7X3vQT+C/kJr02ATcVu1+96PMTQZ/WBh/mk/Lq/1PQ543ARXnlqfoMAP9AbuhmLbA6+Lm4lF/vbvqcmNdbZ+SKiGRIWoZ3RESkABT0RUQyREFfRCRDFPRFRDJEQV9EJEMU9EVEMkRBX0QkQxT0RUQy5P8DXLi3ZYHheSYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.scatter(iterations, train_losses, label='train loss')\n",
    "plt.scatter(iterations, val_losses, label='validation loss')\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [],
   "source": [
    "def q3_acc(real, pred):\n",
    "    if real.shape[0] == 1:\n",
    "        real = np.squeeze(real, 0)\n",
    "    if pred.shape[0] == 1:\n",
    "        pred = np.squeeze(pred, 0)   \n",
    "    return np.sum(real==pred)/real.shape[0]\n",
    "\n",
    "\n",
    "def segment_of_overlap(ss_ref_str, ss_pred_str):\n",
    "    ss_ref = get_segments(ss_ref_str)\n",
    "    ss_pred = get_segments(ss_pred_str)\n",
    "    val_total = 0\n",
    "    N_total = 0\n",
    "    for k in ss_ref.keys():\n",
    "        subsum_k = 0\n",
    "        N_k = 0\n",
    "        for i in ss_ref[k]:\n",
    "            l_s1 = len(i[1])\n",
    "            N_k += l_s1\n",
    "            for j in ss_pred[k]:\n",
    "                l_s2 = len(j[1])\n",
    "                minov = len(np.intersect1d(i[1], j[1]))\n",
    "                if minov > 0:\n",
    "                    maxov = len(np.union1d(i[1], j[1]))\n",
    "                    delta = np.min([maxov-minov, minov, np.floor(l_s1/2), np.floor(l_s2/2)])\n",
    "                    value = l_s1*(minov+delta)/maxov\n",
    "                    subsum_k += value\n",
    "        N_total += N_k\n",
    "        val_total += subsum_k\n",
    "    sov = val_total/N_total\n",
    "    return sov\n",
    "\n",
    "def get_segments(ss_str):\n",
    "    ss_strs = [\"\".join(grp) for val, grp in itertools.groupby(ss_str)]\n",
    "    idx_lens = []\n",
    "    idx_start = 0\n",
    "    for i in ss_strs:\n",
    "        idx_lens.append(np.arange(idx_start, idx_start+len(i)))\n",
    "        idx_start += len(i)\n",
    "    segment_types = {'E':[], 'H':[], '-':[]}\n",
    "    for i in range(len(ss_strs)):\n",
    "        segment_types[ss_strs[i][0]].append([ss_strs[i], idx_lens[i]])\n",
    "    return segment_types\n",
    "\n",
    "def ints_to_symbols1d(ss_arr, map_dict=pos_ss_dict):\n",
    "    if len(ss_arr.shape) > 1:\n",
    "        for i,j in enumerate(ss_arr.shape):\n",
    "            if j == 1:\n",
    "                ss_arr = np.squeeze(ss_arr, i)\n",
    "    return \"\".join([map_dict[i] for i in ss_arr])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6919364164405691"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_accuracies = [q3_acc(i,j) for i,j in zip(validation_preds, validation_reals)]\n",
    "np.mean(val_accuracies)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_preds = [np.apply_along_axis(np.argmax, 1, net(i[0]).detach().numpy()) for i in testloader]\n",
    "test_reals = [np.apply_along_axis(np.argmax, 1, i[1].detach().numpy()) for i in testloader]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6891413061948936"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_accuracies_fcn = [q3_acc(i,j) for i,j in zip(test_preds, test_reals)]\n",
    "np.mean(test_accuracies_fcn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parse_psipred(prot_id, letter_to_number=ss_id_dict):\n",
    "    ss = ''\n",
    "    with open('/Users/Deathvoodoo/big_folders_docs/ss_pred/ss_predictions_psipred/{}.horiz'.format(prot_id)) as input:\n",
    "        lines = input.readlines()\n",
    "        for line in lines:\n",
    "            line = line.strip()\n",
    "            line = line.split()\n",
    "            if len(line)>0:\n",
    "                if line[0] == 'Pred:' and len(line)>1:\n",
    "                    ss += line[1]\n",
    "\n",
    "    ss = ss.replace('C', '-') # hyphens should be equivalent to coils\n",
    "    ss = np.array([letter_to_number[i] for i in ss])\n",
    "    return ss\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "psipred_preds = [parse_psipred(i) for i in test_ids_filt]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "#[i.shape[0]/np.squeeze(j, 0).shape[0] for i,j in zip(psipred_preds, test_reals)] # sanity check for correct order"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "psipred_acc = [q3_acc(i,j) for i,j in zip(psipred_preds, test_reals)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7255188449781512"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.mean(psipred_acc)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's check segment of overlap score."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_preds_str_fcn = [ints_to_symbols1d(i) for i in test_preds]\n",
    "test_preds_str_psipred = [ints_to_symbols1d(i) for i in psipred_preds]\n",
    "test_reals_str = [ints_to_symbols1d(i) for i in test_reals] # same as grabbing strings directly from the dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "FCN SOV:  0.7489448165290328\n",
      "psipred SOV:  0.7850154998573716\n"
     ]
    }
   ],
   "source": [
    "test_sovs_fcn = [segment_of_overlap(i, j) for i,j in zip(test_preds_str_fcn, test_reals_str)]\n",
    "test_sovs_psipred = [segment_of_overlap(i, j) for i,j in zip(test_preds_str_psipred, test_reals_str)]\n",
    "\n",
    "print('FCN SOV: ', np.mean(test_sovs_fcn))\n",
    "print('psipred SOV: ', np.mean(test_sovs_psipred))\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.7.4 64-bit",
   "language": "python",
   "name": "python37464bitdff059f72f8b417fb86b0d43a0194990"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}