Assignment_7.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Assignment_7.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "X02OsnT4aePc",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# http://pytorch.org/\n",
        "from os.path import exists\n",
        "from wheel.pep425tags import get_abbr_impl, get_impl_ver, get_abi_tag\n",
        "platform = '{}{}-{}'.format(get_abbr_impl(), get_impl_ver(), get_abi_tag())\n",
        "cuda_output = !ldconfig -p|grep cudart.so|sed -e 's/.*\\.\\([0-9]*\\)\\.\\([0-9]*\\)$/cu\\1\\2/'\n",
        "accelerator = cuda_output[0] if exists('/dev/nvidia0') else 'cpu'\n",
        "\n",
        "!pip install -q http://download.pytorch.org/whl/{accelerator}/torch-0.4.1-{platform}-linux_x86_64.whl torchvision\n",
        "import torch"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LQ4AfI_taW8D",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "import torchvision.transforms as transforms\n",
        "import torchvision.datasets as dsets\n",
        "from torch.autograd import Variable\n",
        "import numpy as np\n",
        "import math\n",
        "from torch.nn import functional as F"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mMlyfFWhapTy",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "'''\n",
        "STEP 1 - LOADING DATASET\n",
        "\n",
        "'''\n",
        "\n",
        "train_dataset = dsets.MNIST(root='./data', train=True, transform=transforms.ToTensor(), download= True)\n",
        "test_dataset = dsets.MNIST(root='./data', train=False, transform=transforms.ToTensor(), download= True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2DHrtNU3crFU",
        "colab_type": "code",
        "outputId": "af795195-2202-4fe0-e6f1-bc5f06b7a760",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "'''\n",
        "STEP 2 - MAKING DATASET ITERABLE\n",
        "\n",
        "'''\n",
        "\n",
        "batch_size= 100\n",
        "n_iters= 6000\n",
        "num_epochs = n_iters/(len(train_dataset)/batch_size)\n",
        "num_epochs= int(num_epochs)\n",
        "\n",
        "cuda = torch.cuda.is_available()\n",
        "print(cuda)\n",
        "\n",
        "train_loader= torch.utils.data.DataLoader(dataset=train_dataset,batch_size=batch_size, shuffle=True)\n",
        "test_loader= torch.utils.data.DataLoader(dataset=test_dataset,batch_size=batch_size, shuffle= False)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "True\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EqX--saO7_32",
        "colab_type": "text"
      },
      "source": [
        "LSTM class for nn.lstm implementation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oI124Ce7a1Hq",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class LSTM(nn.Module):\n",
        "  def __init__(self, input_size, hidden_size, bias=True):\n",
        "    super(LSTM, self).__init__()\n",
        "    self.input_size = input_size\n",
        "    self.hidden_size = hidden_size\n",
        "    self.bias = bias\n",
        "    if cuda:\n",
        "      self.weight_ih = nn.Parameter(torch.Tensor(4 * hidden_size, input_size)).cuda()\n",
        "      self.weight_hh = nn.Parameter(torch.Tensor(4 * hidden_size, hidden_size)).cuda()\n",
        "    else:\n",
        "      self.weight_ih = nn.Parameter(torch.Tensor(4 * hidden_size, input_size))\n",
        "      self.weight_hh = nn.Parameter(torch.Tensor(4 * hidden_size, hidden_size))\n",
        "    if bias:\n",
        "      if cuda:\n",
        "        self.bias_ih = nn.Parameter(torch.Tensor(4 * hidden_size)).cuda()\n",
        "        self.bias_hh = nn.Parameter(torch.Tensor(4 * hidden_size)).cuda()\n",
        "      else:\n",
        "        self.bias_ih = nn.Parameter(torch.Tensor(4 * hidden_size))\n",
        "        self.bias_hh = nn.Parameter(torch.Tensor(4 * hidden_size))\n",
        "    else:\n",
        "      self.register_parameter('bias_ih', None)\n",
        "      self.register_parameter('bias_hh', None)\n",
        "    self.reset_parameters()\n",
        "\n",
        "  def reset_parameters(self):\n",
        "    stdv = 1.0 / math.sqrt(self.hidden_size)\n",
        "    for weight in self.parameters():\n",
        "      weight.data.uniform_(-stdv, stdv)\n",
        "    \n",
        "  def forward(self,input, hidden):\n",
        "    if input.cuda:\n",
        "        igates = F.linear(input, self.weight_ih)\n",
        "        hgates = F.linear(hidden[0], self.weight_hh)\n",
        "\n",
        "    hx, cx = hidden\n",
        "    gates_1 = F.linear(input, self.weight_ih, self.bias_ih) \n",
        "    gates_2= F.linear(hx, self.weight_hh, self.bias_hh)\n",
        "    print(gates_1.shape, gates_2.shape)\n",
        "    #gates_1 = torch.reshape(gates_1,(28,100,400))\n",
        "    gates = gates_1+gates_2\n",
        "\n",
        "    ingate, forgetgate, cellgate, outgate = gates.chunk(4, 1)\n",
        "\n",
        "    ingate = torch.sigmoid(ingate)\n",
        "    forgetgate = torch.sigmoid(forgetgate)\n",
        "    cellgate = torch.tanh(cellgate)\n",
        "    outgate = torch.sigmoid(outgate)\n",
        "\n",
        "    cy = (forgetgate * cx) + (ingate * cellgate)\n",
        "    hy = outgate * torch.tanh(cy)\n",
        "\n",
        "    return out,(hy, cy)\n",
        "  \n",
        "  \n",
        "  "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2F179Dip8K9o",
        "colab_type": "text"
      },
      "source": [
        "GRU class for nn.Gru implemenation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "73L35ei87Elu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class GRU(nn.Module):\n",
        "  def __init__(self, input_size, hidden_size, bias=True):\n",
        "    super(GRU, self).__init__()\n",
        "    self.input_size = input_size\n",
        "    self.hidden_size = hidden_size\n",
        "    self.bias = bias\n",
        "    if cuda:\n",
        "      self.weight_ih = nn.Parameter(torch.Tensor(3 * hidden_size, input_size)).cuda()\n",
        "      self.weight_hh = nn.Parameter(torch.Tensor(3 * hidden_size, hidden_size)).cuda()\n",
        "    else:\n",
        "      self.weight_ih = nn.Parameter(torch.Tensor(3 * hidden_size, input_size))\n",
        "      self.weight_hh = nn.Parameter(torch.Tensor(3 * hidden_size, hidden_size))\n",
        "    if bias:\n",
        "      if cuda:\n",
        "        self.bias_ih = nn.Parameter(torch.Tensor(3 * hidden_size)).cuda()\n",
        "        self.bias_hh = nn.Parameter(torch.Tensor(3 * hidden_size)).cuda()\n",
        "      else:\n",
        "        self.bias_ih = nn.Parameter(torch.Tensor(3 * hidden_size))\n",
        "        self.bias_hh = nn.Parameter(torch.Tensor(3 * hidden_size))\n",
        "    else:\n",
        "      self.register_parameter('bias_ih', None)\n",
        "      self.register_parameter('bias_hh', None)\n",
        "    self.reset_parameters()\n",
        "\n",
        "  def reset_parameters(self):\n",
        "    stdv = 1.0 / math.sqrt(self.hidden_size)\n",
        "    for weight in self.parameters():\n",
        "      weight.data.uniform_(-stdv, stdv)\n",
        "    \n",
        "  def forward(self,input, hidden):\n",
        "    if input.is_cuda:\n",
        "        gi = F.linear(input, weight_ih)\n",
        "        gh = F.linear(hidden, weight_hh)\n",
        "\n",
        "    gi = F.linear(input, weight_ih, bias_ih)\n",
        "    gh = F.linear(hidden, weight_hh, bias_hh)\n",
        "    i_r, i_i, i_n = gi.chunk(3, 1)\n",
        "    h_r, h_i, h_n = gh.chunk(3, 1)\n",
        "\n",
        "    resetgate = F.sigmoid(i_r + h_r)\n",
        "    inputgate = F.sigmoid(i_i + h_i)\n",
        "    newgate = F.tanh(i_n + resetgate * h_n)\n",
        "    hy = newgate + inputgate * (hidden - newgate)\n",
        "\n",
        "    return hy\n",
        "  "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yeAlmkLOd7XF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "'''\n",
        "STEP 3 - CREATE MODEL CLASS\n",
        "\n",
        "'''\n",
        "\n",
        "class LSTMModel(nn.Module):\n",
        "  def __init__(self, input_dim, hidden_dim, layer_dim, output_dim):\n",
        "    super(LSTMModel, self).__init__()\n",
        "    #hidden dimensions/neurons\n",
        "    self.hidden_dim=hidden_dim\n",
        "    \n",
        "    #no. of hidden layers\n",
        "    self.layer_dim = layer_dim\n",
        "    \n",
        "    #output dimensions/neurons\n",
        "    self.output_dim=output_dim\n",
        "    \n",
        "    # Building your LSTM\n",
        "    # batch_first=True causes input/output tensors to be of shape\n",
        "    # (batch_dim, seq_dim, feature_dim)\n",
        "    #self.lstm = nn.LSTM(input_dim, hidden_dim, layer_dim, batch_first=True)\n",
        "    self.lstm = LSTM(input_dim, hidden_dim, bias=True)\n",
        "    #Readout layer\n",
        "    self.fc = nn.Linear(hidden_dim, output_dim)\n",
        "    \n",
        "    \n",
        "  def forward(self,x):\n",
        "    # Initialize hidden state with zeros\n",
        "        #######################\n",
        "        #  USE GPU FOR MODEL  #\n",
        "        #######################\n",
        "        #print(x.shape,\"x.shape\")100, 28, 28\n",
        "        if torch.cuda.is_available():\n",
        "            h0 = Variable(torch.zeros(self.layer_dim, x.size(0), self.hidden_dim).cuda())\n",
        "        else:\n",
        "            h0 = Variable(torch.zeros(self.layer_dim, x.size(0), self.hidden_dim))\n",
        "         \n",
        "        # Initialize cell state\n",
        "        if torch.cuda.is_available():\n",
        "            c0 = Variable(torch.zeros(self.layer_dim, x.size(0), self.hidden_dim).cuda())\n",
        "        else:\n",
        "            c0 = Variable(torch.zeros(self.layer_dim, x.size(0), self.hidden_dim))\n",
        "        \n",
        "        # One time step\n",
        "        out, (hn, cn) = self.lstm.forward(x, (h0,c0))#or None!\n",
        "        \n",
        "        #[b_ig | b_fg | b_gg | b_og]\n",
        "        #print(self.lstm._all_weights) [['weight_ih_l0', 'weight_hh_l0', 'bias_ih_l0', 'bias_hh_l0'], ['weight_ih_l1', 'weight_hh_l1', 'bias_ih_l1', 'bias_hh_l1'], ['weight_ih_l2', 'weight_hh_l2', 'bias_ih_l2', 'bias_hh_l2']]\n",
        "        for names in self.lstm._all_weights:\n",
        "            for name in filter(lambda n: \"bias\" in n,  names):\n",
        "                bias = getattr(self.lstm, name)\n",
        "                n = bias.size(0)\n",
        "                start, end = n//4, n//2\n",
        "                bias.data[start:end].fill_(1.)\n",
        "\n",
        "        # Index hidden state of last time step\n",
        "        # out.size() --> 100, 28, 100\n",
        "        # out[:, -1, :] --> 100, 100 --> just want last time step hidden states! \n",
        "        out = self.fc(out[:, -1, :]) \n",
        "        # out.size() --> 100, 10\n",
        "        return out"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ll7cqLvHfyld",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "'''\n",
        "STEP 4 - INSTANTIATE MODEL CLASS\n",
        "\n",
        "'''\n",
        "\n",
        "input_dim = 28\n",
        "hidden_dim = 100\n",
        "output_dim = 10\n",
        "layer_dim = 3\n",
        "\n",
        "model = LSTMModel(input_dim, hidden_dim, layer_dim, output_dim)\n",
        "\n",
        "#######################\n",
        "#  USE GPU FOR MODEL  #\n",
        "#######################\n",
        "\n",
        "#if cuda:\n",
        " # model.cuda()\n",
        "\n",
        "'''\n",
        "STEP 5 - INSTANTIATE LOSS CLASS\n",
        "\n",
        "'''\n",
        "criterion = nn.CrossEntropyLoss()\n",
        "\n",
        "'''\n",
        "STEP 6 -INSTANTIATE OPTIMIZER CLASS\n",
        "\n",
        "'''\n",
        "learning_rate= 0.1\n",
        "optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)\n",
        "\n",
        "'''\n",
        "STEP 7 - TRAIN THE MODEL\n",
        "\n",
        "'''\n",
        "#no. f steps to unroll\n",
        "seq_dim = 28\n",
        "\n",
        "iter = 0\n",
        "for epoch in range(num_epochs):\n",
        "  for i, (images, labels) in enumerate(train_loader):\n",
        "    # Load images as Variable\n",
        "    #######################\n",
        "    #  USE GPU FOR MODEL  #\n",
        "    #######################\n",
        "    \n",
        "    if cuda:\n",
        "      images=Variable(images.view(-1, seq_dim, input_dim).cuda())\n",
        "      labels=Variable(labels.cuda())    \n",
        "    else:\n",
        "      images=Variable(images.view(-1, seq_dim, input_dim))\n",
        "      labels=Variable(labels)\n",
        "      \n",
        "    # clear gradients wrt parameters\n",
        "    optimizer.zero_grad()\n",
        "    \n",
        "    # Forward pass to get outputs/logits\n",
        "    # outputs.size() --> 100, 10\n",
        "    outputs=model(images)\n",
        "    \n",
        "    # Calculate Loss: softmax --> cross entropy loss\n",
        "    loss=criterion(outputs,labels)\n",
        "    \n",
        "    # Getting gradients w.r.t. parameters\n",
        "    loss.backward()\n",
        "    \n",
        "    # Updating parameters\n",
        "    optimizer.step()\n",
        "    \n",
        "    iter+=1\n",
        "    if iter%500==0:\n",
        "      #Calculate Accuracy\n",
        "      correct = 0\n",
        "      total = 0\n",
        "      #Iterate through test dataset\n",
        "      for images,labels in test_loader:\n",
        "        #######################\n",
        "        #  USE GPU FOR MODEL  #\n",
        "        #######################\n",
        "        if cuda:\n",
        "          images=Variable(images.view(-1, seq_dim, input_dim).cuda())\n",
        "        else:\n",
        "          images=Variable(images.view(-1, seq_dim, input_dim))\n",
        "        \n",
        "        # Forward pass only to get logits/output\n",
        "        outputs = model(images)\n",
        "        \n",
        "        # Get predictions from the maximum value\n",
        "        _,predicted = torch.max(outputs.data,1)\n",
        "        \n",
        "        # Total no. of labels\n",
        "        total += labels.size(0)\n",
        "        \n",
        "        # Total correct predictions\n",
        "        #######################\n",
        "        #  USE GPU FOR MODEL  #\n",
        "        #######################\n",
        "        \n",
        "        if cuda:\n",
        "          correct += (predicted.cpu()==labels.cpu()).sum()\n",
        "        else:\n",
        "          correct += (predicted.cpu()==labels.cpu())\n",
        "       \n",
        "      accuracy = 100 * correct / total\n",
        "        \n",
        "      # Print Loss\n",
        "      print('Iteration: {}. Loss: {}. Accuracy: {}'.format(iter, loss.data[0], accuracy))\n",
        "        \n",
        "          \n",
        "      \n",
        "    \n",
        "\n",
        "  "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9InQVZAJrBmp",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}