LSTM Sorter.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from dataset import load_svhn, random_split_train_val\n",
    "from layers import *\n",
    "from trainer import *\n",
    "from optim import *\n",
    "from loss import *\n",
    "from collections import OrderedDict\n",
    "from layers import *\n",
    "from operators import *\n",
    "from model import NumberSortModule\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def one_hot(x: (np.ndarray, int), vocab_size: int) -> np.ndarray:\n",
    "    assert isinstance(vocab_size, int), \"vocab_size is an integer value\"\n",
    "    assert isinstance(x, (int, np.ndarray)), \"unsupported type for one-hot encoding\"\n",
    "    if isinstance(x, int):\n",
    "        assert x < vocab_size, \"out of vocabulary\"\n",
    "        y = np.zeros(vocab_size)\n",
    "        y[x] = 1\n",
    "        return y\n",
    "    \n",
    "    assert x.dtype == np.int32, \"unsupported x.dtype for one-hot encoding\"\n",
    "    \n",
    "    y = np.eye(vocab_size)[x.ravel()]\n",
    "    y = y.reshape((*x.shape, vocab_size))\n",
    "    \n",
    "    return y\n",
    "\n",
    "\n",
    "def split_test_train(X, y, test_size=0.25):\n",
    "    train_size = int(len(X) * (1 - test_size))\n",
    "\n",
    "    train_X = X[:train_size, :]\n",
    "    train_y = y[:train_size, :]\n",
    "\n",
    "    test_X = X[train_size:, :]\n",
    "    test_y = y[train_size:, :]\n",
    "\n",
    "    return train_X, train_y, test_X, test_y\n",
    "\n",
    "\n",
    "def generate_batch(batch_size: int = 32, seq_len: int = 10, max_num: int = 100) -> np.ndarray:\n",
    "    while True:\n",
    "        X = np.empty((batch_size, seq_len), dtype=np.int32)\n",
    "        y = np.empty((batch_size, seq_len), dtype=np.int32)\n",
    "\n",
    "        for batch_num in range(batch_size):\n",
    "            sample, label = next(generate_sample_pointer(seq_len, max_num))\n",
    "            X[batch_num] = sample\n",
    "            y[batch_num] = label\n",
    "\n",
    "        yield one_hot(X, vocab_size=max_num), one_hot(y, vocab_size=max_num)\n",
    "\n",
    "\n",
    "def generate_sample_pointer(seq_len: int = 10, max_num: int = 100) -> np.ndarray:\n",
    "    while True:\n",
    "        X = np.random.randint(max_num, size=(seq_len))\n",
    "#         y = np.empty((2, seq_len), dtype=np.int32)\n",
    "#         y[0] = X\n",
    "#         y[1] = np.arange(seq_len)\n",
    "#         y = y[:, y[0].argsort()]\n",
    "        y = np.sort(X, axis=-1)\n",
    "\n",
    "        yield X, y # [1]\n",
    "\n",
    "\n",
    "def create_dataset(num_samples: int, seq_len, max_num) -> Dataset:\n",
    "    data, labels = next(generate_batch(num_samples, seq_len, max_num))\n",
    "    return Dataset(*split_test_train(data, labels))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "x, y = next(generate_batch(1, 5, 10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[[1., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
       "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 1.],\n",
       "        [1., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
       "        [0., 0., 0., 1., 0., 0., 0., 0., 0., 0.],\n",
       "        [0., 0., 0., 0., 0., 0., 1., 0., 0., 0.]]])"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0, 9, 0, 3, 6]])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.argmax(axis=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[[1., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
       "        [1., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
       "        [0., 0., 0., 1., 0., 0., 0., 0., 0., 0.],\n",
       "        [0., 0., 0., 0., 0., 0., 1., 0., 0., 0.],\n",
       "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 1.]]])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0, 0, 3, 6, 9]])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y.argmax(axis=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "vocab_size = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "nsort = NumberSortModule(vocab_size, 5, 10, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = Adam(nsort, learning_rate=1e-4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0001"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "optimizer.learning_rate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "# optimizer.decay(0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = create_dataset(10000, 5, vocab_size)\n",
    "\n",
    "trainer = Trainer(\n",
    "    nsort,\n",
    "    dataset,\n",
    "    optimizer,\n",
    "#     MomentumSGD(nsort, momentum=0.85, learning_rate=0.01),\n",
    "    loss='cross-entropy',\n",
    "    num_epochs=100,\n",
    "    batch_size=1,\n",
    "#     learning_rate_decay=0.85\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss: 2.032206, Train accuracy: 0.321040, val accuracy: 0.315280\n",
      "Loss: 2.029379, Train accuracy: 0.320720, val accuracy: 0.316400\n",
      "Loss: 2.025215, Train accuracy: 0.323413, val accuracy: 0.319040\n",
      "Loss: 2.020979, Train accuracy: 0.324747, val accuracy: 0.315760\n",
      "Loss: 2.018250, Train accuracy: 0.326987, val accuracy: 0.319680\n",
      "Loss: 2.016169, Train accuracy: 0.326773, val accuracy: 0.320160\n",
      "Loss: 2.015247, Train accuracy: 0.328347, val accuracy: 0.320640\n",
      "Loss: 2.015099, Train accuracy: 0.327787, val accuracy: 0.321760\n",
      "Loss: 2.011712, Train accuracy: 0.330507, val accuracy: 0.326560\n",
      "Loss: 2.008553, Train accuracy: 0.336853, val accuracy: 0.331440\n",
      "Loss: 2.005055, Train accuracy: 0.343200, val accuracy: 0.337520\n",
      "Loss: 2.000763, Train accuracy: 0.348187, val accuracy: 0.344320\n",
      "Loss: 1.996293, Train accuracy: 0.352027, val accuracy: 0.344880\n",
      "Loss: 1.992351, Train accuracy: 0.352400, val accuracy: 0.346240\n",
      "Loss: 1.989316, Train accuracy: 0.350373, val accuracy: 0.346160\n",
      "Loss: 1.987608, Train accuracy: 0.352453, val accuracy: 0.344880\n",
      "Loss: 1.985013, Train accuracy: 0.350533, val accuracy: 0.342640\n",
      "Loss: 1.981634, Train accuracy: 0.348320, val accuracy: 0.341600\n",
      "Loss: 1.978306, Train accuracy: 0.347867, val accuracy: 0.342400\n",
      "Loss: 1.975571, Train accuracy: 0.345867, val accuracy: 0.342640\n",
      "Loss: 1.971488, Train accuracy: 0.350907, val accuracy: 0.346160\n",
      "Loss: 1.967919, Train accuracy: 0.350507, val accuracy: 0.345760\n",
      "Loss: 1.964382, Train accuracy: 0.352613, val accuracy: 0.347920\n",
      "Loss: 1.960553, Train accuracy: 0.353600, val accuracy: 0.348080\n",
      "Loss: 1.957713, Train accuracy: 0.355600, val accuracy: 0.350400\n",
      "Loss: 1.956161, Train accuracy: 0.357227, val accuracy: 0.350080\n",
      "Loss: 1.952972, Train accuracy: 0.355733, val accuracy: 0.349680\n",
      "Loss: 1.953118, Train accuracy: 0.358827, val accuracy: 0.353120\n",
      "Loss: 1.952661, Train accuracy: 0.356267, val accuracy: 0.350720\n",
      "Loss: 1.951522, Train accuracy: 0.354427, val accuracy: 0.349120\n",
      "Loss: 1.948587, Train accuracy: 0.356560, val accuracy: 0.351920\n",
      "Loss: 1.946458, Train accuracy: 0.357067, val accuracy: 0.353040\n",
      "Loss: 1.944396, Train accuracy: 0.358560, val accuracy: 0.354480\n",
      "Loss: 1.945768, Train accuracy: 0.359040, val accuracy: 0.354240\n",
      "Loss: 1.945131, Train accuracy: 0.357840, val accuracy: 0.353280\n",
      "Loss: 1.943234, Train accuracy: 0.356187, val accuracy: 0.351040\n",
      "Loss: 1.940323, Train accuracy: 0.355707, val accuracy: 0.351360\n",
      "Loss: 1.939178, Train accuracy: 0.354133, val accuracy: 0.349360\n",
      "Loss: 1.938955, Train accuracy: 0.353093, val accuracy: 0.350400\n",
      "Loss: 1.937241, Train accuracy: 0.355147, val accuracy: 0.350880\n",
      "Loss: 1.935740, Train accuracy: 0.352987, val accuracy: 0.350800\n",
      "Loss: 1.933495, Train accuracy: 0.352853, val accuracy: 0.350640\n",
      "Loss: 1.931952, Train accuracy: 0.352987, val accuracy: 0.349200\n",
      "Loss: 1.930575, Train accuracy: 0.352667, val accuracy: 0.346320\n",
      "Loss: 1.930155, Train accuracy: 0.350027, val accuracy: 0.346320\n",
      "Loss: 1.932122, Train accuracy: 0.350507, val accuracy: 0.345040\n",
      "Loss: 1.933503, Train accuracy: 0.349360, val accuracy: 0.346240\n",
      "Loss: 1.932870, Train accuracy: 0.350027, val accuracy: 0.347280\n",
      "Loss: 1.931821, Train accuracy: 0.353760, val accuracy: 0.349520\n",
      "Loss: 1.930118, Train accuracy: 0.354693, val accuracy: 0.352320\n",
      "Loss: 1.927763, Train accuracy: 0.356987, val accuracy: 0.352480\n",
      "Loss: 1.920085, Train accuracy: 0.362160, val accuracy: 0.359840\n",
      "Loss: 1.912134, Train accuracy: 0.365467, val accuracy: 0.361040\n",
      "Loss: 1.907755, Train accuracy: 0.367653, val accuracy: 0.364080\n",
      "Loss: 1.904759, Train accuracy: 0.368480, val accuracy: 0.364560\n",
      "Loss: 1.903306, Train accuracy: 0.370693, val accuracy: 0.368800\n",
      "Loss: 1.902485, Train accuracy: 0.368773, val accuracy: 0.365360\n",
      "Loss: 1.900427, Train accuracy: 0.371040, val accuracy: 0.368080\n",
      "Loss: 1.899643, Train accuracy: 0.371973, val accuracy: 0.369520\n",
      "Loss: 1.898106, Train accuracy: 0.373227, val accuracy: 0.369120\n",
      "Loss: 1.897881, Train accuracy: 0.374293, val accuracy: 0.370880\n",
      "Loss: 1.899301, Train accuracy: 0.376107, val accuracy: 0.371440\n",
      "Loss: 1.901656, Train accuracy: 0.376853, val accuracy: 0.373520\n",
      "Loss: 1.908866, Train accuracy: 0.376613, val accuracy: 0.373840\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-80-513732ef758f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mloss_history\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_history\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mval_history\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/jb_internships/DL_for_NLP/trainer.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, print_log)\u001b[0m\n\u001b[1;32m    103\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0md_loss_value_accum\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    104\u001b[0m                 \u001b[0mbatch_loss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbatch_loss\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 105\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    106\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    107\u001b[0m                 \u001b[0mbatch_losses\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_loss\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/jb_internships/DL_for_NLP/optim.py\u001b[0m in \u001b[0;36mstep\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     67\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     68\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0moptimizer\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlocalOptimizers\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 69\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_update\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moptimizer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     70\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     71\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_update\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/jb_internships/DL_for_NLP/optim.py\u001b[0m in \u001b[0;36m_update\u001b[0;34m(self, optimizer)\u001b[0m\n\u001b[1;32m     71\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_update\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     72\u001b[0m         \u001b[0;32massert\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moptimizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_LocalOptim\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 73\u001b[0;31m         \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvelocity\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbeta1\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvelocity\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbeta1\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparameter\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgrad\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     74\u001b[0m         \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maccumulated\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbeta2\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maccumulated\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbeta2\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparameter\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgrad\u001b[0m \u001b[0;34m**\u001b[0m \u001b[0;36m2\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     75\u001b[0m         \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madaptive_learning_rate\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlearning_rate\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msqrt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maccumulated\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "loss_history, train_history, val_history = trainer.fit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = Adam(nsort, learning_rate=1e-3)\n",
    "dataset = create_dataset(10000, 5, vocab_size)\n",
    "trainer = Trainer(\n",
    "    nsort,\n",
    "    dataset,\n",
    "    optimizer,\n",
    "    loss='cross-entropy',\n",
    "    num_epochs=100,\n",
    "    batch_size=1,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss: 1.868970, Train accuracy: 0.422187, val accuracy: 0.420160\n",
      "Loss: 1.868968, Train accuracy: 0.421893, val accuracy: 0.420560\n",
      "Loss: 1.868732, Train accuracy: 0.422293, val accuracy: 0.420880\n",
      "Loss: 1.868810, Train accuracy: 0.422267, val accuracy: 0.421120\n",
      "Loss: 1.869100, Train accuracy: 0.421920, val accuracy: 0.422080\n",
      "Loss: 1.869649, Train accuracy: 0.421413, val accuracy: 0.422720\n",
      "Loss: 1.869956, Train accuracy: 0.421627, val accuracy: 0.424400\n",
      "Loss: 1.869928, Train accuracy: 0.421253, val accuracy: 0.423760\n",
      "Loss: 1.870269, Train accuracy: 0.420720, val accuracy: 0.423760\n",
      "Loss: 1.870766, Train accuracy: 0.421067, val accuracy: 0.423680\n",
      "Loss: 1.870982, Train accuracy: 0.421280, val accuracy: 0.423200\n",
      "Loss: 1.871045, Train accuracy: 0.422373, val accuracy: 0.423360\n",
      "Loss: 1.871054, Train accuracy: 0.422613, val accuracy: 0.421840\n",
      "Loss: 1.871179, Train accuracy: 0.422773, val accuracy: 0.422400\n",
      "Loss: 1.871144, Train accuracy: 0.422293, val accuracy: 0.422480\n",
      "Loss: 1.871189, Train accuracy: 0.423120, val accuracy: 0.422400\n",
      "Loss: 1.871457, Train accuracy: 0.423413, val accuracy: 0.422880\n",
      "Loss: 1.871971, Train accuracy: 0.424107, val accuracy: 0.423600\n",
      "Loss: 1.872326, Train accuracy: 0.424080, val accuracy: 0.422880\n",
      "Loss: 1.872440, Train accuracy: 0.423973, val accuracy: 0.422880\n",
      "Loss: 1.872549, Train accuracy: 0.423413, val accuracy: 0.422000\n",
      "Loss: 1.872603, Train accuracy: 0.423493, val accuracy: 0.422640\n",
      "Loss: 1.872455, Train accuracy: 0.423360, val accuracy: 0.422800\n",
      "Loss: 1.872430, Train accuracy: 0.423547, val accuracy: 0.423760\n",
      "Loss: 1.872294, Train accuracy: 0.424053, val accuracy: 0.422160\n",
      "Loss: 1.872500, Train accuracy: 0.423627, val accuracy: 0.423120\n",
      "Loss: 1.872449, Train accuracy: 0.423600, val accuracy: 0.422960\n",
      "Loss: 1.872390, Train accuracy: 0.423840, val accuracy: 0.423200\n",
      "Loss: 1.872673, Train accuracy: 0.423573, val accuracy: 0.422000\n",
      "Loss: 1.873241, Train accuracy: 0.423787, val accuracy: 0.421840\n",
      "Loss: 1.873531, Train accuracy: 0.423653, val accuracy: 0.421760\n",
      "Loss: 1.874171, Train accuracy: 0.422907, val accuracy: 0.421200\n",
      "Loss: 1.874632, Train accuracy: 0.422560, val accuracy: 0.421360\n",
      "Loss: 1.874795, Train accuracy: 0.422027, val accuracy: 0.420080\n",
      "Loss: 1.875071, Train accuracy: 0.422027, val accuracy: 0.418800\n",
      "Loss: 1.875518, Train accuracy: 0.420480, val accuracy: 0.418800\n",
      "Loss: 1.875660, Train accuracy: 0.420400, val accuracy: 0.418960\n",
      "Loss: 1.875447, Train accuracy: 0.420000, val accuracy: 0.419440\n",
      "Loss: 1.875685, Train accuracy: 0.419520, val accuracy: 0.420320\n",
      "Loss: 1.875896, Train accuracy: 0.419520, val accuracy: 0.419920\n",
      "Loss: 1.876007, Train accuracy: 0.419013, val accuracy: 0.420080\n",
      "Loss: 1.876894, Train accuracy: 0.418267, val accuracy: 0.418800\n",
      "Loss: 1.877339, Train accuracy: 0.417467, val accuracy: 0.417520\n",
      "Loss: 1.877550, Train accuracy: 0.418107, val accuracy: 0.418080\n",
      "Loss: 1.878439, Train accuracy: 0.417227, val accuracy: 0.417040\n",
      "Loss: 1.878959, Train accuracy: 0.416080, val accuracy: 0.417040\n",
      "Loss: 1.879176, Train accuracy: 0.416533, val accuracy: 0.417040\n",
      "Loss: 1.879564, Train accuracy: 0.416373, val accuracy: 0.416160\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-92-513732ef758f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mloss_history\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_history\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mval_history\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/jb_internships/DL_for_NLP/trainer.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, print_log)\u001b[0m\n\u001b[1;32m     89\u001b[0m                 \u001b[0md_loss_value_accum\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     90\u001b[0m                 \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msample\u001b[0m \u001b[0;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 91\u001b[0;31m                     \u001b[0mpred\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msample\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     92\u001b[0m                     \u001b[0mpred\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpred\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpred\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpred\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     93\u001b[0m                     \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloss\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'mse'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/jb_internships/DL_for_NLP/model.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, x_input)\u001b[0m\n\u001b[1;32m     47\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     48\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0msample\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 49\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_reload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     50\u001b[0m             \u001b[0mlstm_encoder_out\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlstm_encoder\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx_input_aligned\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0msample\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     51\u001b[0m             \u001b[0;31m# print(\"LSTM output\", lstm_encoder_out)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/jb_internships/DL_for_NLP/model.py\u001b[0m in \u001b[0;36m_reload\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     93\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_reload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     94\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mn_layers\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 95\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlstm_encoder\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     96\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     97\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0mlstm\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlstm_encoder\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodules\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/jb_internships/DL_for_NLP/layers.py\u001b[0m in \u001b[0;36mreload\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    297\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mreload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    298\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep_num\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 299\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhistory\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    300\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    301\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "loss_history, train_history, val_history = trainer.fit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer.decay(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "unsorted:  [8 2 4 2 9]\n",
      "sorted:    [2 2 4 8 9]\n",
      "predicted: [2 2 4 8 8]\n"
     ]
    }
   ],
   "source": [
    "rand_index = random.randint(0, dataset.val_y.shape[0] - 1)\n",
    "testX = dataset.val_X[rand_index]\n",
    "testY = dataset.val_y[rand_index]\n",
    "test_num = testX.argmax(axis=-1).ravel()\n",
    "srtd = np.eye(testX.shape[-1])[np.sort(test_num)]\n",
    "predicted = nsort.predict(testX).ravel()\n",
    "predicted_forward = nsort.forward(testX)\n",
    "predicted_forward = predicted_forward.reshape(predicted_forward.shape[-2], predicted_forward.shape[-1])\n",
    "predicted_onehot = np.eye(testX.shape[-1])[predicted_forward.argmax(axis=-1)]\n",
    "print(\"unsorted: \", test_num)\n",
    "print(\"sorted:   \", np.sort(test_num))\n",
    "print(\"predicted:\", predicted)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}