encoder_decoder_batch.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import unicode_literals, print_function, division\n",
    "from io import open\n",
    "import unicodedata\n",
    "import string\n",
    "import re\n",
    "import random\n",
    "import numpy as np\n",
    "import pickle\n",
    "import time\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch import optim\n",
    "import torch.nn.functional as F\n",
    "import pdb\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "plt.switch_backend('agg')\n",
    "%matplotlib inline\n",
    "\n",
    "from model_architectures import Encoder_RNN, Decoder_RNN\n",
    "from data_prep import prepareData, tensorsFromPair, prepareNonTrainDataForLanguagePair, load_cpickle_gc\n",
    "from inference import generate_translation\n",
    "from misc import timeSince, load_cpickle_gc\n",
    "\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "!export CUDA_LAUNCH_BLOCKING=1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Constants"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "BATCH_SIZE = 32\n",
    "PAD_token = 0\n",
    "SOS_token = 1\n",
    "EOS_token = 2\n",
    "UNK_token = 3\n",
    "MAX_LENGTH = 30\n",
    "teacher_forcing_ratio = 1.0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Data prep - train and val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reading lines...\n",
      "['khoa hoc ang sau mot tieu e ve khi hau', 'rachel pike the science behind a climate headline']\n",
      "Read 133317 sentence pairs\n",
      "Trimmed to 133317 sentence pairs\n",
      "Counting words...\n",
      "['khoa hoc ang sau mot tieu e ve khi hau', 'rachel pike the science behind a climate headline']\n",
      "Counted words:\n",
      "vi 14237\n",
      "en 41272\n"
     ]
    }
   ],
   "source": [
    "input_lang, target_lang, train_pairs = prepareData(\n",
    "    input_file = 'iwslt-vi-en/train.tok.vi', \n",
    "    target_file = 'iwslt-vi-en/train.tok.en', \n",
    "    input_lang = 'vi', \n",
    "    target_lang = 'en')\n",
    "\n",
    "pickle.dump(train_pairs, open(\"preprocessed_data_no_elmo/iwslt-vi-eng/preprocessed_no_indices_pairs\", \"wb\"))\n",
    "pickle.dump(input_lang, open(\"preprocessed_data_no_elmo/iwslt-vi-eng/preprocessed_no_elmo_vilang\", \"wb\"))\n",
    "pickle.dump(target_lang, open(\"preprocessed_data_no_elmo/iwslt-vi-eng/preprocessed_no_elmo_englang\", \"wb\"))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_lang = pickle.load(open(\"preprocessed_data_no_elmo/iwslt-vi-eng/preprocessed_no_elmo_vilang\", \"rb\"))\n",
    "target_lang = pickle.load(open(\"preprocessed_data_no_elmo/iwslt-vi-eng/preprocessed_no_elmo_englang\", \"rb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "non_train_pairs = prepareNonTrainDataForLanguagePair(\n",
    "    input_file_path_dev = 'iwslt-vi-en/dev.tok.vi', \n",
    "    target_file_path_dev = 'iwslt-vi-en/dev.tok.en',\n",
    "    input_file_path_test = 'iwslt-vi-en/test.tok.vi', \n",
    "    target_file_path_test = 'iwslt-vi-en/test.tok.en',\n",
    "    input_lang = input_lang, \n",
    "    target_lang = target_lang)\n",
    "# this function pickles to vi-envalidation_tokenized and vi-entest_tokenized \n",
    "val_pairs = non_train_pairs[0]\n",
    "test_pair = non_train_pairs[1]\n",
    "# now dump these. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "pickle.dump(val_pairs, open(\"preprocessed_data_no_elmo/iwslt-vi-eng/preprocessed_no_indices_pairs_validation_tokenized\", \"wb\"))\n",
    "pickle.dump(test_pair, open(\"preprocessed_data_no_elmo/iwslt-vi-eng/preprocessed_no_indices_pairs_test_tokenized\", \"wb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_pairs = pickle.load(open(\"preprocessed_data_no_elmo/iwslt-vi-eng/preprocessed_no_indices_pairs\", \"rb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--Return--\n",
      "> <ipython-input-10-3f910cd0b97e>(5)<module>()->None\n",
      "-> pdb.set_trace()\n",
      "(Pdb) train_idx_pairs[0]\n",
      "[tensor([[ 4],\n",
      "        [ 5],\n",
      "        [ 6],\n",
      "        [ 7],\n",
      "        [ 8],\n",
      "        [ 9],\n",
      "        [10],\n",
      "        [11],\n",
      "        [12],\n",
      "        [13],\n",
      "        [ 2]]), tensor([[ 4],\n",
      "        [ 5],\n",
      "        [ 6],\n",
      "        [ 7],\n",
      "        [ 8],\n",
      "        [ 9],\n",
      "        [10],\n",
      "        [11],\n",
      "        [ 2]])]\n",
      "(Pdb) n\n",
      "> /anaconda2/envs/nlpclass/lib/python3.6/site-packages/IPython/core/interactiveshell.py(2964)run_code()\n",
      "-> sys.excepthook = old_excepthook\n",
      "(Pdb) c\n"
     ]
    }
   ],
   "source": [
    "train_idx_pairs = []\n",
    "for x in train_pairs:\n",
    "    indexed = list(tensorsFromPair(x, input_lang, target_lang))\n",
    "    train_idx_pairs.append(indexed)\n",
    "pdb.set_trace()\n",
    "pickle.dump(train_idx_pairs, open(\"preprocessed_data_no_elmo/iwslt-vi-eng/preprocessed_no_indices_pairs_train_tokenized\", \"wb\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Laboratory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import torch\n",
    "from torch.utils.data import Dataset\n",
    "\n",
    "class LanguagePairDataset(Dataset):\n",
    "    \n",
    "    def __init__(self, sent_pairs): \n",
    "        # this is a list of sentences \n",
    "        self.sent_pairs_list = sent_pairs\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.sent_pairs_list)\n",
    "        \n",
    "    def __getitem__(self, key):\n",
    "        \"\"\"\n",
    "        Triggered when you call dataset[i]\n",
    "        \"\"\"\n",
    "        sent1 = self.sent_pairs_list[key][0][:MAX_LENGTH]\n",
    "        sent2 = self.sent_pairs_list[key][1][:MAX_LENGTH]\n",
    "        return [sent1, sent2, len(sent1), len(sent2)]\n",
    "\n",
    "def language_pair_dataset_collate_function(batch):\n",
    "    \"\"\"\n",
    "    Customized function for DataLoader that dynamically pads the batch so that all \n",
    "    data have the same length\n",
    "    \"\"\"\n",
    "    sent1_list = []\n",
    "    sent1_length_list = []\n",
    "    sent2_list = []\n",
    "    sent2_length_list = []\n",
    "    # padding\n",
    "    for datum in batch:\n",
    "        padded_vec_1 = np.pad(np.array(datum[0]).T.squeeze(), pad_width=((0,MAX_LENGTH-len(datum[0]))), \n",
    "                                mode=\"constant\", constant_values=PAD_token)\n",
    "        padded_vec_2 = np.pad(np.array(datum[1]).T.squeeze(), pad_width=((0,MAX_LENGTH-len(datum[1]))), \n",
    "                                mode=\"constant\", constant_values=PAD_token)\n",
    "        sent1_list.append(padded_vec_1)\n",
    "        sent2_list.append(padded_vec_2)\n",
    "        sent1_length_list.append(len(datum[0]))\n",
    "        sent2_length_list.append(len(datum[1]))\n",
    "    print(np.array(sent1_list).shape)\n",
    "    return [torch.from_numpy(np.array(sent1_list)), torch.LongTensor(sent1_length_list), \n",
    "            torch.from_numpy(np.array(sent2_list)), torch.LongTensor(sent2_length_list)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "#train_idx_pairs = load_cpickle_gc(\"train_vi_en_idx_pairs\")\n",
    "train_dataset = LanguagePairDataset(train_idx_pairs)\n",
    "train_loader = torch.utils.data.DataLoader(dataset=train_dataset, \n",
    "                                           batch_size=BATCH_SIZE, \n",
    "                                           collate_fn=language_pair_dataset_collate_function,\n",
    "                                           #shuffle=True\n",
    "                                          )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(32, 30)\n",
      "0\n",
      "torch.Size([32, 30])\n",
      "torch.Size([32, 30])\n",
      "torch.Size([32])\n",
      "torch.Size([32])\n"
     ]
    }
   ],
   "source": [
    "for i, (sent1_list, sent1_lengths, sent2_list, sent2_lengths) in enumerate(train_loader):\n",
    "    print(i)\n",
    "    print(sent1_list.size())\n",
    "    print(sent2_list.size())\n",
    "    \n",
    "    print(sent1_lengths.size())\n",
    "    print(sent2_lengths.size())\n",
    "    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Encoder_Batch_RNN(nn.Module):\n",
    "    def __init__(self, input_size, hidden_size):\n",
    "        super(Encoder_Batch_RNN, self).__init__()\n",
    "        self.hidden_size = hidden_size\n",
    "        self.embedding = nn.Embedding(input_size, hidden_size)\n",
    "        self.gru = nn.GRU(hidden_size, hidden_size, batch_first=True)\n",
    "        \n",
    "    def init_hidden(self, batch_size):\n",
    "        return torch.zeros(1, batch_size, self.hidden_size, device=device)\n",
    "\n",
    "    def forward(self, sents, sent_lengths):\n",
    "        '''\n",
    "            sents is a tensor with the shape (batch_size, padded_length )\n",
    "        '''\n",
    "        batch_size = sents.size()[0]\n",
    "        sent_lengths = list(sent_lengths)\n",
    "        # We sort and then do pad packed sequence here. \n",
    "        descending_lengths = [x for x, _ in sorted(zip(sent_lengths, range(len(sent_lengths))), reverse=True)]\n",
    "        descending_indices = [x for _, x in sorted(zip(sent_lengths, range(len(sent_lengths))), reverse=True)]\n",
    "        descending_lengths = np.array(descending_lengths)\n",
    "        \n",
    "        descending_sents = torch.index_select(sents, 0, torch.tensor(descending_indices).to(device))\n",
    "        \n",
    "        # get embedding\n",
    "        embed = self.embedding(descending_sents)\n",
    "        # pack padded sequence\n",
    "        embed = torch.nn.utils.rnn.pack_padded_sequence(embed, descending_lengths, batch_first=True)\n",
    "        \n",
    "        # fprop though RNN\n",
    "        self.hidden = self.init_hidden(batch_size)\n",
    "        rnn_out, self.hidden = self.gru(embed, self.hidden)\n",
    "        \n",
    "        # change the order back\n",
    "        change_it_back = [x for _, x in sorted(zip(descending_indices, range(len(descending_indices))))]\n",
    "        self.hidden = torch.index_select(self.hidden, 1, torch.LongTensor(change_it_back).to(device)) \n",
    "        \n",
    "        # **TODO**: What is rnn_out?\n",
    "        return rnn_out, self.hidden"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Decoder_Batch_RNN(nn.Module):\n",
    "    def __init__(self, output_size, hidden_size):\n",
    "        super(Decoder_Batch_RNN, self).__init__()\n",
    "        self.hidden_size = hidden_size\n",
    "\n",
    "        self.embedding = nn.Embedding(output_size, hidden_size)\n",
    "        self.gru = nn.GRU(hidden_size, hidden_size, batch_first=True)\n",
    "        self.out = nn.Linear(hidden_size, output_size)\n",
    "        self.softmax = nn.LogSoftmax(dim=1)\n",
    "        \n",
    "    def init_hidden(self, batch_size):\n",
    "        return torch.zeros(1, batch_size, self.hidden_size, device=device)\n",
    "\n",
    "    def forward(self, sents, sent_lengths, hidden):\n",
    "        \n",
    "        batch_size = sents.size()[0]\n",
    "        sent_lengths = list(sent_lengths)\n",
    "        \n",
    "        descending_lengths = [x for x, _ in sorted(zip(sent_lengths, range(len(sent_lengths))), reverse=True)]\n",
    "        descending_indices = [x for _, x in sorted(zip(sent_lengths, range(len(sent_lengths))), reverse=True)]\n",
    "        descending_lengths = np.array(descending_lengths)\n",
    "        \n",
    "        descending_sents = torch.index_select(sents, 0, torch.tensor(descending_indices).to(device))\n",
    "        \n",
    "        # get embedding\n",
    "        embed = self.embedding(descending_sents)\n",
    "        # pack padded sequence\n",
    "        embed = torch.nn.utils.rnn.pack_padded_sequence(embed, descending_lengths, batch_first=True)\n",
    "        \n",
    "        # fprop though RNN\n",
    "        self.hidden = hidden\n",
    "        rnn_out, self.hidden = self.gru(embed, self.hidden)\n",
    "        \n",
    "        change_it_back = [x for _, x in sorted(zip(descending_indices, range(len(descending_indices))))]\n",
    "        self.hidden = torch.index_select(self.hidden, 1, torch.LongTensor(change_it_back).to(device))\n",
    "        rnn_out, _ = torch.nn.utils.rnn.pad_packed_sequence(rnn_out, batch_first=True)\n",
    "        # rnn_out is batch_size x 28 x 256\n",
    "                \n",
    "        final_hidden = self.hidden\n",
    "        final_hidden = final_hidden.view(final_hidden.size(1), final_hidden.size(0), -1)\n",
    "        first_hidden = hidden\n",
    "        first_hidden = first_hidden.view(first_hidden.size(1), first_hidden.size(0), -1)\n",
    "        \n",
    "        rnn_out = torch.cat((first_hidden, rnn_out, final_hidden), 1)\n",
    "        \n",
    "        \n",
    "#         rnn_out = rnn_out.view(-1, rnn_out.size(2))\n",
    "        \n",
    "        output = self.softmax(self.out(rnn_out))\n",
    "        # now output is the size 28 by 31257 (vocab size)\n",
    "        return output, hidden"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(0.9869)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = torch.randn(3, 4)\n",
    "a[0][0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The whole training process looks like this:\n",
    "\n",
    "-  Start a timer\n",
    "-  Initialize optimizers and criterion\n",
    "-  Create set of training pairs\n",
    "-  Start empty losses array for plotting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "def trainIters(encoder, decoder, n_iters, n_epochs, pairs, validation_pairs, lang1, lang2, print_every=1000, plot_every=100, learning_rate=0.001):\n",
    "    \"\"\"\n",
    "    lang1 is the Lang object for language 1 \n",
    "    Lang2 is the Lang object for language 2\n",
    "    \"\"\"\n",
    "    start = time.time()\n",
    "    plot_losses = []\n",
    "    print_loss_total = 0  # Reset every print_every\n",
    "    plot_loss_total = 0  # Reset every plot_every\n",
    "    encoder_optimizer = torch.optim.SGD(encoder.parameters(), lr=learning_rate)\n",
    "    decoder_optimizer = torch.optim.SGD(decoder.parameters(), lr=learning_rate)\n",
    "    \n",
    "    criterion = nn.NLLLoss(ignore_index=PAD_token) # this ignores the padded token. \n",
    "    \n",
    "    for epoch in range(n_epochs):\n",
    "        for step, (sent1s, sent1_lengths, sent2s, sent2_lengths) in enumerate(train_loader):\n",
    "            encoder.train() # what is this for?\n",
    "            sent1_batch, sent2_batch = sent1s.to(device), sent2s.to(device) \n",
    "            sent1_length_batch, sent2_length_batch = sent1_lengths.to(device), sent2_lengths.to(device)\n",
    "            \n",
    "            encoder_optimizer.zero_grad()\n",
    "            outputs, encoder_hidden = encoder(sent1_batch, sent1_length_batch)\n",
    "            # encoder outputs is currently size 696 x 256\n",
    "            encoder_hidden_batch = encoder_hidden \n",
    "            decoder_hidden = encoder_hidden_batch\n",
    "            \n",
    "            decoder_input = torch.tensor([[SOS_token]], device=device)\n",
    "            use_teacher_forcing = True\n",
    "            \n",
    "            loss = 0\n",
    "            outputs, decoder_hidden = decoder(sent2_batch, sent2_length_batch, decoder_hidden)\n",
    "            \n",
    "            count = 0\n",
    "            # here, we loop through the batch of the generated batch fromd ecoder\n",
    "            # here, we only generate the length of the second batch. \n",
    "            for i in range(len(sent2_batch)):\n",
    "                l = sent2_length_batch[i]\n",
    "                for j in range(l):\n",
    "                    o = outputs[i][j].view(1, -1)\n",
    "                    s = torch.tensor([sent2_batch[i][j]])\n",
    "                    loss += criterion(o, s) # this will ignore if s is \"EOS\"\n",
    "                    count += 1\n",
    "            print_loss_total += loss\n",
    "            plot_loss_total += loss  \n",
    "            \n",
    "            if  (step+1) % print_every == 0:\n",
    "                print_loss_avg = print_loss_total / (count)\n",
    "                print_loss_total = 0\n",
    "                print('TRAIN SCORE %s (%d %d%%) %.4f' % (timeSince(start, step / n_epochs),\n",
    "                                             step, step / n_epochs * 100, print_loss_avg))\n",
    "                val_loss = test_model(encoder, decoder, search, validation_pairs, lang1, max_length=MAX_LENGTH)\n",
    "                # returns bleu score\n",
    "                print(\"VALIDATION BLEU SCORE: \"+str(val_loss))\n",
    "\n",
    "            if step % plot_every == 0:\n",
    "                plot_loss_avg = plot_loss_total / plot_every\n",
    "                plot_losses.append(plot_loss_avg)\n",
    "                plot_loss_total = 0\n",
    "\n",
    "    showPlot(plot_losses)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(32, 30)\n",
      "(32, 30)\n",
      "(32, 30)\n",
      "(32, 30)\n",
      "(32, 30)\n",
      "(32, 30)\n",
      "(32, 30)\n",
      "(32, 30)\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-31-94af17898ade>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     20\u001b[0m }\n\u001b[1;32m     21\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 22\u001b[0;31m \u001b[0mtrainIters\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-30-c7fe621395c2>\u001b[0m in \u001b[0;36mtrainIters\u001b[0;34m(encoder, decoder, n_iters, n_epochs, pairs, validation_pairs, lang1, lang2, print_every, plot_every, learning_rate)\u001b[0m\n\u001b[1;32m     29\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     30\u001b[0m             \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 31\u001b[0;31m             \u001b[0moutputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdecoder_hidden\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdecoder\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msent2_batch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msent2_length_batch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdecoder_hidden\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     32\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     33\u001b[0m             \u001b[0mcount\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda2/envs/nlpclass/lib/python3.6/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    475\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    476\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 477\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    478\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    479\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-16-87d4b3bc6e4d>\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, sents, sent_lengths, hidden)\u001b[0m\n\u001b[1;32m     47\u001b[0m \u001b[0;31m#         rnn_out = rnn_out.view(-1, rnn_out.size(2))\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     48\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 49\u001b[0;31m         \u001b[0moutput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msoftmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrnn_out\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     50\u001b[0m         \u001b[0;31m# now output is the size 28 by 31257 (vocab size)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     51\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0moutput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhidden\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda2/envs/nlpclass/lib/python3.6/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    475\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    476\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 477\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    478\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    479\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda2/envs/nlpclass/lib/python3.6/site-packages/torch/nn/modules/activation.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m    784\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    785\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 786\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlog_softmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdim\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_stacklevel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/anaconda2/envs/nlpclass/lib/python3.6/site-packages/torch/nn/functional.py\u001b[0m in \u001b[0;36mlog_softmax\u001b[0;34m(input, dim, _stacklevel)\u001b[0m\n\u001b[1;32m    973\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mdim\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    974\u001b[0m         \u001b[0mdim\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_get_softmax_dim\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'log_softmax'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdim\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_stacklevel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 975\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlog_softmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdim\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    976\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    977\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "hidden_size = 256\n",
    "#input_lang = load_cpickle_gc(\"train_vi_lang\")\n",
    "#target_lang = load_cpickle_gc(\"train_en_lang\")\n",
    "encoder1 = Encoder_Batch_RNN(input_lang.n_words, hidden_size).to(device)\n",
    "decoder1 = Decoder_Batch_RNN(target_lang.n_words, hidden_size).to(device)\n",
    "\n",
    "num_iters = 10000\n",
    "\n",
    "args = {\n",
    "    'n_iters': 10000,\n",
    "    'n_epochs': 4,\n",
    "    'learning_rate': 0.001,\n",
    "    'encoder': encoder1,\n",
    "    'decoder': decoder1,\n",
    "    'lang1': input_lang, \n",
    "    'lang2': target_lang,\n",
    "    \"pairs\":train_pairs, \n",
    "    \"validation_pairs\": val_pairs, \n",
    "    'print_every': 10\n",
    "}\n",
    "\n",
    "trainIters(**args)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# torch.save(encoder1.state_dict(), \"encoder1_40000\")\n",
    "# torch.save(decoder1.state_dict(), \"decoder1_40000\")\n",
    "\n",
    "pairs = load_cpickle_gc(\"train_\"+input_lang.name+\"_\"+target_lang.name+\"_pairs\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> trong phan lon cuoc oi minh toi chua tung ban khoan ve kha nang phat ngon cua minh .\n",
      "= for most of my life i never gave a second thought to my ability to speak .\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "only one element tensors can be converted to Python scalars",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-27-8da33c7ccfd9>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0mencoder1\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0meval\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0mdecoder1\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0meval\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m \u001b[0mevaluateRandomly\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpairs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mencoder1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdecoder1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m10\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstrategy\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'greedy'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mk\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m8\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-25-e112f6aa4c1e>\u001b[0m in \u001b[0;36mevaluateRandomly\u001b[0;34m(pairs, encoder, decoder, n, strategy, k, max_length)\u001b[0m\n\u001b[1;32m      7\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'>'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpair\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'='\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpair\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 9\u001b[0;31m         \u001b[0moutput_words\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgenerate_translation\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mencoder\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdecoder\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpair\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msearch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mstrategy\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mk\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmax_length\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmax_length\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     10\u001b[0m         \u001b[0moutput_sentence\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m' '\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput_words\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'<'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moutput_sentence\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-24-d0308729bb12>\u001b[0m in \u001b[0;36mgenerate_translation\u001b[0;34m(encoder, decoder, sentence, max_length, search, k)\u001b[0m\n\u001b[1;32m     80\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mei\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput_length\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     81\u001b[0m             encoder_output, encoder_hidden = encoder(input_tensor[ei],\n\u001b[0;32m---> 82\u001b[0;31m                                                      encoder_hidden)\n\u001b[0m\u001b[1;32m     83\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     84\u001b[0m         \u001b[0;31m# start decoding\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    475\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    476\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 477\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    478\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    479\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-6-10800d8e396f>\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, sents, sent_lengths)\u001b[0m\n\u001b[1;32m     18\u001b[0m         \u001b[0mdescending_lengths\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mx\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_\u001b[0m \u001b[0;32min\u001b[0m \u001b[0msorted\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msent_lengths\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msent_lengths\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreverse\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     19\u001b[0m         \u001b[0mdescending_indices\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mx\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0m_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0msorted\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msent_lengths\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msent_lengths\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreverse\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 20\u001b[0;31m         \u001b[0mdescending_lengths\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdescending_lengths\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     21\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m         \u001b[0mdescending_sents\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mindex_select\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msents\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdescending_indices\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: only one element tensors can be converted to Python scalars"
     ]
    }
   ],
   "source": [
    "# hidden_size = 256\n",
    "# encoder1 = EncoderRNN(input_lang.n_words, hidden_size).to(device)\n",
    "# encoder1.load_state_dict(torch.load(\"encoder1_40000\", map_location='cpu'))\n",
    "# decoder1 = DecoderRNN(target_lang.n_words, hidden_size).to(device)\n",
    "# decoder1.load_state_dict(torch.load(\"decoder1_40000\", map_location='cpu'))\n",
    "encoder1.eval()\n",
    "decoder1.eval()\n",
    "evaluateRandomly(pairs, encoder1, decoder1, n = 10, strategy='greedy', k = 8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluateRandomly(pairs, encoder, decoder, n=10, strategy=\"greedy\", k = None, max_length = MAX_LENGTH):\n",
    "    \"\"\"\n",
    "    Randomly select a sentence from the input dataset and try to produce its translation.\n",
    "    \"\"\"    \n",
    "    for i in range(n):\n",
    "        pair = random.choice(pairs)\n",
    "        print('>', pair[0])\n",
    "        print('=', pair[1])\n",
    "        output_words = generate_translation(encoder, decoder, pair[0], search=strategy, k = k, max_length = max_length)\n",
    "        output_sentence = ' '.join(output_words)\n",
    "        print('<', output_sentence)\n",
    "        print('')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def indexesFromSentence(lang, sentence):\n",
    "    words = sentence.split(' ')\n",
    "    indices = []\n",
    "    for word in words:\n",
    "        if lang.word2index.get(word) is not None:\n",
    "            indices.append(lang.word2index[word])\n",
    "        else:\n",
    "            indices.append(1) # UNK_INDEX\n",
    "    return indices\n",
    "\n",
    "def tensorFromSentence(lang, sentence):\n",
    "    indexes = indexesFromSentence(lang, sentence)\n",
    "    indexes.append(EOS_token)\n",
    "    return torch.tensor(indexes, dtype=torch.long, device=device).view(-1, 1)\n",
    "\n",
    "def greedy_search(decoder, decoder_input, hidden, max_length):\n",
    "    translation = []\n",
    "    for i in range(max_length):\n",
    "        next_word_softmax, hidden = decoder(decoder_input, hidden)\n",
    "        best_idx = torch.max(next_word_softmax, 1)[1].squeeze().item()\n",
    "\n",
    "        # convert idx to word\n",
    "        best_word = target_lang.index2word[best_idx]\n",
    "        translation.append(best_word)\n",
    "        decoder_input = torch.tensor([[best_idx]], device=device)\n",
    "        \n",
    "        if best_word == 'EOS':\n",
    "            break\n",
    "    return translation\n",
    "\n",
    "\n",
    "def beam_search(decoder, decoder_input, hidden, max_length, k):\n",
    "    \n",
    "    candidates = [(decoder_input, 0, hidden)]\n",
    "    potential_candidates = []\n",
    "    completed_translations = []\n",
    "\n",
    "    # put a cap on the length of generated sentences\n",
    "    for m in range(max_length):\n",
    "        for c in candidates:\n",
    "            # unpack the tuple\n",
    "            c_sequence = c[0]\n",
    "            c_score = c[1]\n",
    "            c_hidden = c[2]\n",
    "            # EOS token\n",
    "            if c_sequence[-1] == 1:\n",
    "                completed_translations.append((c_sequence, c_score))\n",
    "                k = k - 1\n",
    "            else:\n",
    "                next_word_probs, hidden = decoder(c_sequence[-1], c_hidden)\n",
    "                # in the worst-case, one sequence will have the highest k probabilities\n",
    "                # so to save computation, only grab the k highest_probability from each candidate sequence\n",
    "                top_probs, top_idx = torch.topk(next_word_probs, k)\n",
    "                for i in range(len(top_probs[0])):\n",
    "                    word = torch.from_numpy(np.array([top_idx[0][i]]).reshape(1, 1)).to(device)\n",
    "                    new_score = c_score + top_probs[0][i]\n",
    "                    potential_candidates.append((torch.cat((c_sequence, word)).to(device), new_score, hidden))\n",
    "\n",
    "        candidates = sorted(potential_candidates, key= lambda x: x[1], reverse=True)[0:k] \n",
    "        potential_candidates = []\n",
    "\n",
    "    completed = completed_translations + candidates\n",
    "    completed = sorted(completed, key= lambda x: x[1], reverse=True)[0] \n",
    "    final_translation = []\n",
    "    for x in completed[0]:\n",
    "        final_translation.append(target_lang.index2word[x.squeeze().item()])\n",
    "    return final_translation\n",
    "\n",
    "def generate_translation(encoder, decoder, sentence, max_length, search=\"greedy\", k = None):\n",
    "    \"\"\" \n",
    "    @param max_length: the max # of words that the decoder can return\n",
    "    @returns decoded_words: a list of words in target language\n",
    "    \"\"\"    \n",
    "    with torch.no_grad():\n",
    "        input_tensor = tensorFromSentence(input_lang, sentence)\n",
    "        input_length = input_tensor.size()[0]\n",
    "        \n",
    "        # encode the source sentence\n",
    "        encoder_hidden = encoder.init_hidden(1)\n",
    "        for ei in range(input_length):\n",
    "            encoder_output, encoder_hidden = encoder(input_tensor[ei],\n",
    "                                                     encoder_hidden)\n",
    "\n",
    "        # start decoding\n",
    "        decoder_input = torch.tensor([[SOS_token]], device=device)  # SOS\n",
    "        decoder_hidden = encoder_hidden\n",
    "        decoded_words = []\n",
    "        \n",
    "        if search == 'greedy':\n",
    "            decoded_words = greedy_search(decoder, decoder_input, decoder_hidden, max_length)\n",
    "        elif search == 'beam':\n",
    "            if k == None:\n",
    "                k = 2\n",
    "            decoded_words = beam_search(decoder, decoder_input, decoder_hidden, max_length, k)  \n",
    "\n",
    "        return decoded_words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate(encoder, decoder, sentence, search=\"greedy\", max_length=MAX_LENGTH):\n",
    "    \"\"\"\n",
    "    Function that generate translation.\n",
    "    First, feed the source sentence into the encoder and obtain the hidden states from encoder.\n",
    "    Secondly, feed the hidden states into the decoder and unfold the outputs from the decoder.\n",
    "    Lastly, for each outputs from the decoder, collect the corresponding words in the target language's vocabulary.\n",
    "    And collect the attention for each output words.\n",
    "    @param encoder: the encoder network\n",
    "    @param decoder: the decoder network\n",
    "    @param sentence: string, a sentence in source language to be translated\n",
    "    @param max_length: the max # of words that the decoder can return\n",
    "    @output decoded_words: a list of words in target language\n",
    "    @output decoder_attentions: a list of vector, each of which sums up to 1.0\n",
    "    \"\"\"    \n",
    "    # process input sentence\n",
    "    with torch.no_grad():\n",
    "        input_tensor = tensorFromSentence(input_lang, sentence)\n",
    "        input_length = input_tensor.size()[0]\n",
    "        # encode the source lanugage\n",
    "        encoder_hidden = encoder.initHidden()\n",
    "\n",
    "        encoder_outputs = torch.zeros(max_length, encoder.hidden_size, device=device)\n",
    "\n",
    "        for ei in range(input_length):\n",
    "            encoder_output, encoder_hidden = encoder(input_tensor[ei],\n",
    "                                                     encoder_hidden)\n",
    "            encoder_outputs[ei] += encoder_output[0, 0]\n",
    "        decoder_input = torch.tensor([[SOS_token]], device=device)  # SOS\n",
    "        # decode the context vector\n",
    "        decoder_hidden = encoder_hidden # decoder starts from the last encoding sentence\n",
    "        # output of this function\n",
    "        decoder_attentions = torch.zeros(max_length, max_length)\n",
    "        \n",
    "        if search == 'greedy':\n",
    "            decoded_words = greedy_search(decoder, decoder_input, decoder_hidden, max_length)\n",
    "        elif search == 'beam':\n",
    "            decoded_words = beam_search(decoder, decoder_input, decoder_hidden, max_length)  \n",
    "        return decoded_words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sacrebleu\n",
    "def calculate_bleu(predictions, labels):\n",
    "\t\"\"\"\n",
    "\tOnly pass a list of strings \n",
    "\t\"\"\"\n",
    "\t# tthis is ony with n_gram = 4\n",
    "\n",
    "\tbleu = sacrebleu.raw_corpus_bleu(predictions, [labels], .01).score\n",
    "\treturn bleu"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_e = EncoderRNN(5551, 256)\n",
    "model_e.load_state_dict(torch.load(\"encoder1_40000\", map_location='cpu'))\n",
    "model_e.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_d = DecoderRNN(19344, 256)\n",
    "model_d.load_state_dict(torch.load(\"decoder1_40000\", map_location='cpu'))\n",
    "model_d.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "MAX_LENGTH = 100\n",
    "def test_model(encoder, decoder, search, test_pairs, lang1, max_length=MAX_LENGTH):\n",
    "    # for test, you only need the lang1 words to be tokenized,\n",
    "    # lang2 words is the true labels\n",
    "    encoder_inputs = [pair[0] for pair in test_pairs]\n",
    "    true_labels = [pair[1] for pair in test_pairs]\n",
    "    translated_predictions = []\n",
    "    for i in range(len(encoder_inputs)): \n",
    "        if i% 100== 0:\n",
    "            print(i)\n",
    "        e_input = encoder_inputs[i]\n",
    "        decoded_words = generate_translation(encoder, decoder, e_input, max_length=MAX_LENGTH)\n",
    "        translated_predictions.append(\" \".join(decoded_words))\n",
    "    return calculate_bleu(translated_predictions, true_labels)\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "TODO: \n",
    "    \n",
    "Yikes, teh decoder isn't' preforming very well. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_pairs = pickle.load(open(\"preprocessed_data_no_elmo/iwslt-vi-eng/preprocessed_no_indices_pairs_test\", \"rb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_lang = pickle.load(open(\"preprocessed_data_no_elmo/iwslt-vi-eng/preprocessed_no_elmo_vilang\", \"rb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_model(model_e, model_d, \"greedy\", test_pairs, input_lang )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}