LanguagesModelsDataGenerator.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "from mxnet import nd, gluon, init, context, autograd\n",
    "from mxnet.gluon import nn\n",
    "\n",
    "import re\n",
    "import collections\n",
    "import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_time_machine():\n",
    "    with open('./data/timemachine.txt', 'r') as f:\n",
    "        lines = f.readlines()\n",
    "    return [re.sub('[^A-Za-z]+', ' ', line.strip().lower()) for line in lines]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['the time machine by h g wells ', '', '', '', '']\n"
     ]
    }
   ],
   "source": [
    "lines = read_time_machine()\n",
    "print(lines[:5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tokenize(lines):\n",
    "    tokens = [line.split(' ') for line in lines]\n",
    "    return tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['the', 'time', 'machine', 'by', 'h', 'g', 'wells', ''], [''], [''], [''], ['']]\n"
     ]
    }
   ],
   "source": [
    "tokens = tokenize(lines)\n",
    "print(tokens[:5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Vocab(object):\n",
    "    def __init__(self, tokens, min_freq=0):\n",
    "        \n",
    "        self.token_freqs = Vocab.count_freqs(tokens)\n",
    "        self.token_freqs = sorted(self.token_freqs.items(), key=lambda x: x[0])\n",
    "        self.token_freqs = sorted(self.token_freqs, key=lambda x:x[1], reverse=True)\n",
    "        \n",
    "        self.unk, self.uniq_tokens = 0, ['<unk>']\n",
    "        self.uniq_tokens += [token for token, freq in self.token_freqs if freq > min_freq]\n",
    "        \n",
    "        self.idx_to_token, self.token_to_idx = [], dict()\n",
    "        for token in self.uniq_tokens:\n",
    "            self.idx_to_token.append(token)\n",
    "            self.token_to_idx[token] = len(self.idx_to_token) - 1\n",
    "        \n",
    "    @staticmethod\n",
    "    def count_freqs(tokens):\n",
    "        flat = [tk for line in tokens for tk in line]\n",
    "        return collections.Counter(flat)\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.idx_to_token)\n",
    "    \n",
    "    \n",
    "    def __getitem__(self, tokens):\n",
    "        if not isinstance(tokens, (list, tuple)):\n",
    "            return self.token_to_idx[tokens]\n",
    "        return [self.token_to_idx[token] for token in tokens]\n",
    "    def to_tokens(self, indices):\n",
    "        if not isinstance(tokens, (list, tuple)):\n",
    "            return self.idx_to_token[indices]\n",
    "        return [self.idx_to_token[index] for index in indices]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "vocab = Vocab(tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('the', 2261), ('', 1282), ('i', 1267), ('and', 1245), ('of', 1155), ('a', 816), ('to', 695), ('was', 552), ('in', 541), ('that', 443)]\n"
     ]
    }
   ],
   "source": [
    "print(vocab.token_freqs[:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7f54a0bd14a8>]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deXxU5d3+8c93JishCYQEgbAnrFIBjVA3SqsiVhG3KrWtVVGqVWsfW1tr26f+qt0361algrj00SpWBRWpWMWlVgiIyCKLrGGXQFhDtvv3R4ZphCRkmcmZM3O9X6+8yNyzcHEMXN5nuY855xAREQEIeB1ARERih0pBRETCVAoiIhKmUhARkTCVgoiIhKkUREQkLMnrAK2Rm5vrevfu7XUMERFfWbBgwafOubz6nvN1KfTu3Zvi4mKvY4iI+IqZrW/oOV/uPjKzcWY2uayszOsoIiJxxZel4Jyb6ZyblJ2d7XUUEZG44stSEBGR6FApiIhImEpBRETCVAoiIhKmUhARkTCVgoiIhKkUREQkTKUgIiJhKgUREQnzZSlomQsRkejwZSlomQsRkejwZSmIiEh0qBRERCRMpSAiImEqBRERCVMpiIhImEpBRETCVAoiIhKmUhARkbAkrwO0xurt+xh//zste7MZQYNgwAiYEQzYZ76v/RWSAgECgdrX1v4aer7u96HXHh5LCj2fFDBSk4KkJgdITQqQkhSofXz419B4alIw9FzoKzlIRkoQM4vsBhMROQZfl0JSwOiYkdKi99Y4qKlxVNc4qp2joqqGaudqx5yjuqb2+aqaGmocta+rcdS4z/5a+z3hz6l9j2v1ny0zNYlBXbMY1DWTwd2yGNQ1i/7HZZKWHGz1Z4uINMSca/0/YF4pKipyxcXFXseo1+FyqKiu4VBlNYeqakJf1VQc/r6y9vGhqprQWO335ZXVbCw9yLIte/h4yx72V1QDEDAoyGsfKossBnfL4oT87BYXo4gkJjNb4Jwrqu85X88UYlkgYKQEjJSkAO1TW76Za2ocG0oPsHzLHpZv2cOyLXtYsH4XMz7cDEB6cpAnJo6gqHdOpKKLSAKLmZmCmQ0CbgFygdedc3851ntieaYQbbsPVLBsyx5+/PwSdu47xPQbTqX/cZlexxIRH2hsphDVs4/MbKqZbTezJUeMjzWzFWa22sxuB3DOLXfOXQ9cBpwWzVzxoEO7FE4tyOXxa0aQlhzkyinz2LT7oNexRMTnon1K6jRgbN0BMwsCDwDnAoOBr5rZ4NBzFwAvA69EOVfc6JHTjseuGcH+Q1VcOeV9du2v8DqSiPhYVEvBOfcWUHrE8AhgtXNujXOuAngaGB96/Qzn3LnA1xr6TDObZGbFZla8Y8eOaEX3lUFds/jrN4vYuOsgV0+bz4GKKq8jiYhPeXHxWj6wsc7jEiDfzEab2b1m9jCNzBScc5Odc0XOuaK8vLxoZ/WNz/ftxL0ThrG4ZDc3/m0hldU1XkcSER+KmSuanXNvOue+45z7lnPuAa/z+NHYIV2568IhvLFiBz98bjGxchKBiPiHF6ekbgJ61HncPTTWZGY2DhhXWFgYyVxx4Wsje/Hp3gr+NGcleZmp/OjcQV5HEhEf8WKmMB/oZ2Z9zCwFmADMaM4H6B7NjfvOmYV8/fM9eXjuGh55e43XcUTER6J9SupTwHvAADMrMbOJzrkq4CZgNrAceMY5tzSaORKNmfH/LhjCuUO6cPfLy/nOUx+wfW+517FExAdi5uK15qiz++i6VatWeR0nZlVU1fDgm6t58I1PSEsOcPu5g5hwcg8CAS20J5LIGrt4zZelcFgiX9HcHJ/s2MePn/+I/6wp5aReHfnlRZ9jQBdd/SySqDy7olliQ0Fee5667vP84StDWbNjH+fd+za/efVjDoYW2hMROcyXpWBm48xscllZmddRfMPMuOSk7rz+vdFcNDyfv7z5CZdPfo+95ZVeRxORGOLLUtDZRy2Xk5HC774ylMnfOIllm/cw8bFizRhEJMyXpSCtN+b4Lvzp8mHMX1fK9U8uoKJKV0CLiEohoY0b2o1fXfQ55q7cwXf//gFVWhpDJOH58iY7uqI5ciaM6Mm+Q1Xc/fJy2qV8xG8vOUGnrIokMF/OFHRMIbKuPaMv3z2rH9MXlHDnzKWU7Dqg3UkiCcqXMwWJvFvO7Mfe8iqmvLOWx99bjxl0ykilS3Yqg7pkcecFx5PRituKiog/6G+5ALWnrP7kvEGcNeg41u/cz9Y95WzbU87m3eU8t7CEsoOVPPT1k7RrSSTOqRQkzMw4paATpxR0+sz4lHfWctdLy/jjayv5/jkDPEonIm3Bl8cUdPFa27rmtN5MOLkH97+xmhcXNWuVcxHxGV+Wgg40ty0z4+fjhzCiTw63TV/MBxt2eR1JRKJEC+JJk5Xur2D8A+9wsKKGC4d1IyM1iYzUIO1SkmifmkS7lCDt05Io6pVDSpIv/39DJCE0tiCejilIk+VkpDDlmydzw5ML+L95GzjQwPIYZw8+jr9eWe/Pm4jEOJWCNEv/4zJ5/XujAaiucRysrObAoSr2V1Sz/1AVMxdv5uG5a5i7cgdf6J/nbVgRaTbN8aXFggGjfWoSnbPS6JObwZD8bG49uz+9O7Xj5zOXUqllM0R8x5eloLOPYldqUpCfnj+YT3bs57F/r/M6jog0ky9LQWcfxbYvDezMF/rn8ec5q9ix95DXcUSkGXxZChLbzIz/HTeYg5XV/PG1FV7HEZFmUClIVBTktedrI3vybHEJG0sPeB1HRJpIpSBRc/3oAgJmPDT3E6+jiEgTqRQkarpmp3NpUXeeLS5ha1m513FEpAlUChJVN3yhgBrnNFsQ8QlfloJOSfWPHjntuPjEfJ6at4HtezVbEIl1viwFnZLqL98eXUhldQ0TpxUzfUEJByqqvI4kIg3wZSmIv/TOzeDXl5zA3vJKvv/sh4z8xevMXbnD61giUg+VgrSJy4p68Mb3R/PMt06he047vvVEMfPWlnodS0SOoFKQNmNmjOiTwxMTR9CtQzoTp81nySYdFxKJJSoFaXO57VN5cuJIstKTuXLqPFZv3+t1JBEJUSmIJ7p1SOfJa0cSMONrj7yvq55FYoRKQTzTJzeDJ68dQXllDZc9/B6/n72CeWtL8fPdAEX8TqUgnhrYJYvHr6k9xvCXuZ9w2cPvce1jxXy6T6urinhB92iWmLGnvJJn5m/kt7NXkJWWxCUnducL/fM4tTDX62gicaWxezRrpiAxIystmWvP6MvMm05nYJcsHn13HVc88j6vLtnqdTSRhOHLUtAyF/FtQJdMnrx2JB/+bAy57VN4+aMtXkcSSRi+LAUtc5EY0lOCnDnwON5csZ2KKt3vWaQt+LIUJHGcNfg49pZX6epnkTaiUpCYdnphLmnJAV5bpuMKIm1BpSAxLT0lyOmFecxZvl3XL4i0AZWCxLyxQ7qwafdBrp42n9Xb93kdRySuqRQk5l08PJ87vjyQBet3cdGD77Jtj27WIxItKgWJeYGAMWlUATNuOp2KqhrunLHU60gicUulIL7RJzeDW87qx6wlW7nt2Q/57asfs2D9Lh1rEImgJK8DiDTHdWf05T9rSpm1ZCvlldU8+OYnDO2ezW8vHcqALplexxPxPa19JL6171AVMz/czO9nr2D3wUpO6tmRy0/uwUXD8wkEzOt4IjGrsbWPVAriezv3HWLqu2t5bdk2Vm7bx8AumQzJzyYtuXbvaH6HdlwwrBv5HdI9TioSG1QKkhBqahzTF5YwfUEJ63fup6ra4YDS/RWYwWkFufzyos/Rs1M7r6OKeEqlIAltY+kBnv9gE4+8vYbqGkfXDukU5GWQlZbMgC6ZXDGyJ+1SdHhNEkdjpRBTfxPM7ELgPCALmOKc+6fHkSQO9Mhpx3fO7Mf4Yd148I1P2H2wglXb97GvvIpnF5SwZFMZ90wY7nVMkZgQ9VIws6nA+cB259yQOuNjgT8DQeAR59yvnXMvAC+YWUfg94BKQSKmV6cMfnPpCZ8Zu2fOSu6Zs4r1pQf41qi+jB3S1aN0IrGhLa5TmAaMrTtgZkHgAeBcYDDwVTMbXOclPwk9LxJVN36xkNvOGcCeg5Vc/+RCrcYqCS/qpeCcews48m/aCGC1c26Nc64CeBoYb7V+A8xyzi2s7/PMbJKZFZtZ8Y4dO6IbXuJecjDAjV8s5KWbzyAzNYmn52/wOpKIp7w6ppAPbKzzuAQYCdwMnAVkm1mhc+6hI9/onJsMTIbaA81tkFUSQHpKkPOHduO5hSVkpCQxtEcHBnbJ5PhuWZjpmgdJHDF1oNk5dy9wr9c5JDHdcmY/du2v4NkFG3niP+sBGDP4OG4d05+BXbI8TifSNrwqhU1AjzqPu4fGmsTMxgHjCgsLI51LEliX7DQe+sZJHKyoZkvZQWYt2cqfX1/FP5dtY2iPDlxyYj45GSkMOC6TvnntCeqqaYlDbXKdgpn1Bl46fPaRmSUBK4EzqS2D+cAVzrlmLX+p6xQk2nbtr+D5Dzbx9PwNrNz233s55LZP4a9XFjG8Z0cP04m0jKcXr5nZU8BoIBfYBvzMOTfFzL4M3EPtKalTnXO/aO5nqxSkrTjn2FB6gP2HqvnPmp1MeWctldU1zP7uKDpmpHgdT6RZ4u6K5jq7j65btWqV13EkAS3ZVMbFD/6bQd2yeHLiCDLTkr2OJNJkcVcKh2mmIF7659Kt3PC3heRkpNC9YzoXD8/nipG9dKxBYl5jpaCb7Ii00Jjju/DExBGc2LMDldU1/PTFpVz84Ls8U7yRJZvKvI4n0iK+nClo95HEGuccMz7czF0vLePTfbWrsk696mS+OKCz19FEjtKqmYKZ/dbMssws2cxeN7MdZvb1yMdsOufcTOfcpOzsbC9jiISZGeOH5fPOD7/EnFtH0S07nfv/tdrrWCLN1pTdR2Occ3uoXdRuHVAI3BbNUCJ+lZYcpLBzJt84pRcL1u9iY+kBryOJNEtTSuHwBW7nAc8657SzVOQYvjykKwGDW59ZROn+Cq/jiDRZU0rhJTP7GDgJeN3M8oDy6MZqnJmNM7PJZWXqJ4lNPTu1488ThrO4pIyiu1/jHwtLKK+s9jqWyDE16UCzmeUAZc65ajPLADKdc1ujnu4YdEqqxLolm8q46tF5fLqvgn6d2zPrljNICuqkP/FWq65TMLOLG3veOfePVmRrFZWC+EHp/goefXct9/1rNSf37sg9E4aT3yHd61iSwFp7O86JwKnAv0KPvwj8G9gBOMCzUhDxg5yMFG49uz/JwQD3zFnJVVPnMf36U8lup6ugJfY0pRSSgcHOuS0AZtYVmOacuzqqyUTiiJnxnTP7UdS7I9+cOo9JTxRz69n9SQoGGNajg66ClpjRlN1Hy51zg+o8DgBL6461NV28Jn724qJN3PL0ovDjAcdlcs+EYQzqqns2SNto7TGF+4F+wFOhocupvZXmzRFN2QI6piB+tWrbXrbvPcT7a3Zyb+git3FDu3H1ab05UctxS5S1ekE8M7sIGBV6+JZz7vkI5msxlYLEg2Wb93DH8x+xaONuAKZdfTKjtTyGRFEkSqEX0M85N8fM2gFB59zeCOdsNpWCxJPNuw8y/oF32bH3EF//fE9+fsEQAjrWIFHQ2rWPrgOmAw+HhvKBFyIXT0QAunVI56nrRjIkP4sn/7OBP762ksrqGq9jSYJpylU0NwKnAXsAnHOrAM1tRaKgsHMmM286nTP65XL/G6sZ8Ys57Nx3yOtYkkCaUgqHnHPhxVtC91f2dL1tLXMh8czMuP+KE/nRuQPZfbCSr0+Zx97ySq9jSYJoSinMNbM7gHQzOxt4FpgZ3ViN09LZEu+y05P51hcKuPTE7izfsofP3flPvjHlfWZ9tIWKKu1SkuhpyimpAWqvah4DGDAbeMTFwN15dKBZ4l1NjeOZ4o08NPcT1u2sXYa7d6d2fP+cAZx/QjeP04lftfjsIzMLAo87574WrXCtoVKQRFFRVcPGXQe466VlvLliBwDdO6bzs3HHc/bg4zxOJ37T4rOPnHPVQC8zS4lKMhFpkpSkAAV57Zl29Qjm3DqKy4t6ULLrINc9Xsxf3vyEAxVVXkeUONGU3UePA4OAGcD+w+POuT9GN9qxaaYgiezNFdu56tH5ABzfLYsXbjyNZC3LLU3QopmCmT0R+vYC4KXQazPrfImIh0YP6My8O86kIC+DpZv38N066ymJtFRjq6SeZGbdgA3AfW2Up0nqLIjndRQRT3XOSmPWLaMY+cs5vPzRFq5eV0pR7xyvY4mPNTbXfAh4HegPFNf5WhD61TM6JVXkv1KSAsy46XQALn3oPe54/iOPE4mfNVgKzrl7Q8tjP+qc61vnq49zrm8bZhSRY+iR044/XT6Uod2z+b/3N3Dxg+9SpSUypAWOeVTKOXdDWwQRkda5aHh3Hvz6SRTkZbBww276/WQWMz/c7HUs8RmdqiASR/I7pPP8jadxx5cH0j41iZuf+oClm7UcjDSdSkEkzmSlJTNpVAF3XzgEgPPufYclm1QM0jQqBZE4NX5YPj8cOxCAK6fOY86ybR4nEj9QKYjEsRtGF/DDsQOpqKrhpy8u4Yq//oc3Vmz3OpbEsMauUxCROHDD6AIA3vh4O4s27uaXLy9n0Ybd3PSlQl0BLUfRT4RIArhhdAHPXH8KV4zoScmug/z59VU88vZa3lixnfLKaq/jSQxp0j2aY02dK5qvW7VqlddxRHxl255yTvnV69SE/urfdeEQvvH5Xt6GkjbV4qWzY50WxBNpmY2lByg7WMnlD7/H/opqstKSmP0/o+iane51NGkDLV46W0TiU4+cdgzJz+YPlw1jwsk92FNexU9fWOp1LIkBKgWRBDZ2SBd+fN4gAOYs38aG0N3dJHGpFEQSXGZaMn/4ylAARv3uDeau3OFxIvGSTkkVEc47oSsA33v2Qx7412pmL93K5/Kz+eqInh4nk7ammYKIkJYc5KLh+YzoncOaT/fz/MJN3PXSMq9jiQdUCiICQCBgPHP9KRT/5CxuPrOQAxXVnHDnbJ5bUOJ1NGlDKgUROcqFw/KZNKovVTWOf328nTU79uHn09el6VQKInKUbh3SuePLg+iTm8HLH23hS3+YqzWTEoQONItIg+6/4kTmryvlB9MXM3fFDgJmpCQFGNE7hyStmxSX9F9VRBrUJzeDC4flk54c5LH31nPVo/O54q/v8+rSrV5HkyjRTEFEGpWSFOC1W0exfe8h9pZX8c2p8/hgw256d8rg+G5ZmJnXESWCNFMQkWPq3rEdJ/bsyKkFnUhNCjDlnbWcf987zFtb6nU0iTCVgog0WXIwwMvfOZ17Lh8GwPqdB3DO6cykOBIzpWBmfc1siplN9zqLiDSssHMmY4d0wQx+8Nxi+vzoFfr86BXue13L2MeDqJaCmU01s+1mtuSI8bFmtsLMVpvZ7QDOuTXOuYnRzCMikZGWHOSey4fx3bP68d2z+nFcViqLNu72OpZEQLQPNE8D7gcePzxgZkHgAeBsoASYb2YznHO6pl7ER8YPyw9/X7xuF0s2l3HnjNrltzNSg9z8pX6kJQe9iictFNVScM69ZWa9jxgeAax2zq0BMLOngfFAk0rBzCYBkwB69tRiXSKx4NTCTiwu2c0/FpZQXePYX1HNKX1zOb1frtfRpJm8OKaQD2ys87gEyDezTmb2EDDczH7U0Judc5Odc0XOuaK8vLxoZxWRJvj26EIW33kOi+88h+dvPA2A3QcrPE4lLREz1yk453YC13udQ0RaJzs9GYD/fXEpv3n1YwBu/lI/Livq4WUsaSIvZgqbgLo/Hd1DY01mZuPMbHJZWVlEg4lI63XOTOXGLxYwun8eJ/fKYff+St5Z9anXsaSJvJgpzAf6mVkfastgAnBFcz7AOTcTmFlUVHRdFPKJSCuYGbedMzD8+Pz73mbfoSoPE0lzRLUUzOwpYDSQa2YlwM+cc1PM7CZgNhAEpjrndMdwkTjVPjWJd1d/yujfvREeCwaMu8YP4dRCHYiONdE+++irDYy/ArzS0s81s3HAuMLCwpZ+hIi0kWtO68PLWVs+M/bios3MW1eqUohBMXOguTm0+0jEP8Yc34Uxx3f5zNisJVs5WFntUSJpjC9LQUT8rV1KkA/W72bKO2s/M94tO41zP9fVo1QCPi0F7T4S8be+uRnMW1fKvHVHr7K6+M4xZKUle5BKwKeloN1HIv72zLdOYX/FZ3cfPVu8kbtfXk55RbVKwUO+LAUR8bekYIDs9M9eJtWhXQoAh6pqvIgkISoFEYkJqUm1JfHI22vIyUgNjycFjcuKepCXmdrQWyWCfFkKOqYgEn/65GaQmhTgsffWH/VcalKAa8/o60GqxOPLUtAxBZH4MyQ/m+U/H/uZsaoaR/+fzOJAhU5fbSu+LAURiU+BgH3mcUrACAaMQ1UqhbYSM7fjFBGpT1pSgPJKHXxuK5opiEhMS0sO8tzCknpXWk0KGndfOIThPTt6kCw++bIUdKBZJHHcMLqA4nW7jhqvqqlhzvLtLFi/S6UQQb4sBR1oFkkc157Rl2vPOHq8vLKagT99lYpq7VqKJB1TEBFfSg7W/vNVWeU8ThJfVAoi4kvB0JlJlZopRJQvdx+JiAAkB41dByrYvPtgg6/p2C6F9JRgG6byN5WCiPhWRkoSf3t/A397f0ODr+mTm8Eb3x/ddqF8zpeloLOPRARg8pUnsXr7vgaff/mjrcxbu7MNE/mfL0tBZx+JCMBJvXI4qVdOg8+v23mAf68++voGaZgONItI3EoOGFU1Dud0hlJTqRREJG4lhU5bra5RKTSVSkFE4lZSsHaBvSqVQpOpFEQkbiUHav+JUyk0nS8PNIuINMXhmcJVU+cRPGJZ7vrkd0zn95cOPWoJ70Tiy5mCmY0zs8llZWVeRxGRGHZKQSdOL8xtUiFs3VPOPxZuYm95VRski13m56PyRUVFrri42OsYIhIHpr27ljtnLmPhT88mJyPF6zhRZWYLnHNF9T3ny5mCiEikBXWmEqBSEBEBIGi1u5hqfLz3JBJUCiIiQGiioJmC1wFERGJBIDRTUCmIiEj4DCXtPhIRkXApaKYgIiLh3UeJPlPQFc0iIvx3pjDro610yd7d7PebGaP659I5My3S0dqUL0tBN9kRkUjrnJkKwB9eW9niz7jq1N7cecHxkYrkCV+Wgm6yIyKRVtQ7h/k/PovyyuoWvf+iB99t8XtjiS9LQUQkGvJCs4WWSAoEiIfDETrQLCISAWbxcZBapSAiEgEBM/xfCSoFEZGI0ExBRETCzNAxBRERqRUww8/3pzlMpSAiEgEGxMMKGSoFEZEICJjpmIKIiNQyQ2cfiYhILdMxBREROSxgUFPjdYrWUymIiERA7cVrmimIiEhIPJx9FDML4plZBvAgUAG86Zz7m8eRRESaTNcpNIGZTTWz7Wa25IjxsWa2wsxWm9ntoeGLgenOueuAC6KZS0Qk0gKB+LiiOdozhWnA/cDjhwfMLAg8AJwNlADzzWwG0B34KPQy/y9KLiIJxTCWbC7j1r8vapPf74qRPSnqnRPxz41qKTjn3jKz3kcMjwBWO+fWAJjZ08B4aguiO7CIRmYwZjYJmATQs2fPyIcWEWmBUf1zmfHhZuavL22T32/M8V2i8rleHFPIBzbWeVwCjATuBe43s/OAmQ292Tk3GZgMUFRUFAeTNRGJB7edM5DbzhnodYxWi5kDzc65/cDVXucQEUlkXpySugnoUedx99BYk5nZODObXFZWFtFgIiKJzotSmA/0M7M+ZpYCTABmNOcDnHMznXOTsrOzoxJQRCRRRfuU1KeA94ABZlZiZhOdc1XATcBsYDnwjHNuaTRziIhI00T77KOvNjD+CvBKSz/XzMYB4woLC1v6ESIiUg9fLnOh3UciItHhy1IQEZHo8GUp6OwjEZHoMD8v4GRmO4D1QDZwZEMcOXbk41zg06gGrD9XpN97rNc19nxDzx1r2x05lijbsrHXRGpbQmxvz+a8r6U/m80Z9/Pfcy+3ZS/nXF69n+Sc8/0XMPlYY/U8LvYiV6Tfe6zXNfZ8Q88da9sdOZYo27I526yl2zLWt2dz3tfSn83mjPv577mX27KxL1/uPqpHfctiHDnW4NIZUdSa37Op7z3W6xp7vqHnmrLt2np7xsK2bOw1ftqWrfk9m/O+lv5sNmfcz3/PvdyWDfL17qPWMLNi51yR1znigbZlZGl7Ro62ZfPFy0yhJSZ7HSCOaFtGlrZn5GhbNlPCzhRERORoiTxTEBGRI6gUREQkTKUgIiJhKoUQM8sws8fM7K9m9jWv8/iZmfU1sylmNt3rLH5nZheGfib/bmZjvM7jZ2Y2yMweMrPpZnaD13liVVyXgplNNbPtZrbkiPGxZrbCzFab2e2h4YuB6c6564AL2jxsjGvOtnTOrXHOTfQmaexr5rZ8IfQzeT1wuRd5Y1kzt+Vy59z1wGXAaV7k9YO4LgVgGjC27oCZBYEHgHOBwcBXzWwwtXeAO3zv6Oo2zOgX02j6tpTGTaP52/Inoefls6bRjG1pZhcAL9OKpfvjXVyXgnPuLaD0iOERwOrQ/81WAE8D44ESaosB4ny7tEQzt6U0ojnb0mr9BpjlnFvY1lljXXN/Lp1zM5xz5wLaRdyARPzHL5//zgigtgzygX8Al5jZX/DmUnk/qndbmlknM3sIGG5mP/Immu809HN5M3AWcKmZXe9FMB9q6OdytJnda2YPo5lCg6J65zU/cc7tB672Okc8cM7tpHYfuLSSc+5e4F6vc8QD59ybwJsex4h5iThT2AT0qPO4e2hMmk/bMnK0LSNH27IVErEU5gP9zKyPmaUAE4AZHmfyK23LyNG2jBxty1aI61Iws6eA94ABZlZiZhOdc1XATcBsYDnwjHNuqZc5/UDbMnK0LSNH2zLytCCeiIiExfVMQUREmkelICIiYSoFEREJUymIiEiYSkFERMJUCiIiEqZSEKmHmXUws28f4zWjzeyltsok0hZUCiL16wA0Wgoi8UilIFK/XwMFZrbIzH4X+lpiZh+Z2VE3uzGzk83sAzMrCN3Fb6qZzQuNjQ+95ioz+4eZvWpmq8zstzrqVn4AAAGNSURBVKHxoJlNq/P5/9PGf1aRMK2SKlK/24EhzrlhZnYJtau+DgVygflm9tbhF5rZqcB9wHjn3AYz+yXwL+fcNWbWAZhnZnNCLx8GDAcOASvM7D6gM5DvnBsS+rwObfRnFDmKZgoix3Y68JRzrto5tw2YC5wcem4QMBkY55zbEBobA9xuZouoXao5DegZeu5151yZc64cWAb0AtYAfc3sPjMbC+xpiz+USH1UCiKtswUop/b//g8z4BLn3LDQV0/n3PLQc4fqvK4aSHLO7aJ2FvImtTOSR6IfW6R+KgWR+u0FMkPfvw1cHtr3nweMAuaFntsNnAf8ysxGh8ZmAzebmQGYWd3COIqZ5QIB59xz1N6L+cRI/kFEmkPHFETq4ZzbaWbvmtkSYBawGPgQcMAPnHNbzWxg6LXbzOx8YJaZXQPcBdwDLDazALAWOL+R3y4feDT0WgDdwlQ8o6WzRUQkTLuPREQkTKUgIiJhKgUREQlTKYiISJhKQUREwlQKIiISplIQEZEwlYKIiIT9f5Pi7G6vWCNOAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "freqs = [freq for token, freq in vocab.token_freqs]\n",
    "axes = plt.gca()\n",
    "axes.set_xlabel('tokens')\n",
    "axes.set_ylabel('freqs')\n",
    "axes.set_xscale('log')\n",
    "axes.set_yscale('log')\n",
    "axes.plot(freqs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "digram_tokens = [[pair for pair in zip(line[:-1], line[1:])] for line in tokens]\n",
    "digram_vocab = Vocab(digram_tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7f5491c567f0>]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3dd3yV5f3/8dfnZEIIYSTsTUCWCIiCC3FjFaGuKm6tWBXbb5fV37ff2mXraq1WW0pRUati3aggKoIoLoao7D3CnmFmnuv3x30SAiQh65z7nJP38/E4j3iu+75PPlxCPrm2OecQEREBCPgdgIiIRA8lBRERKaWkICIipZQURESklJKCiIiUUlIQEZFSiX4HUBuZmZmuU6dOfochIhJT5s6du905l1XetZhOCp06dWLOnDl+hyEiElPMbG1F12Ky+8jMhpvZuNzcXL9DERGJKzGZFJxzbzvnRmdkZPgdiohIXInJpCAiIuGhpCAiIqWUFEREpJSSgoiIlFJSEBGRUkoKIiJSSklBRERKKSmIiEgpJQURESkVk0lB21yIiIRHTCYFbXMhIhIeMZkUREQkPJQURESklJKCiIiUUlIQEZFSSgoiIlJKSUFEREopKYiISCklBRERKZXodwC1sWLrPkY88WnNHjYjMWAkmJEQMBITjECoLBAIXSv7Mu+ehNJnAiQEOOxrYsBISQyQmpRAapL3NSUxgQbJCaSWlh+6lpqUQFpyAokJys0iEh1iOikkBoymack1ejboIBh0FAWDFAcd+UXFFAcdxc5RVOwIOkdR0HllR7yKgi70rHd/SXlNpCQG6NM2g37tm9C/QxP6tW9C2yYNMLMafZ6ISG2YczX7YRYNBg4c6ObMmeN3GAC4UHLILwqSV1hMXujrwYJi8ouKySsMlZd8LfKubcrNY/763SzYkEt+URCAzEYppQmif4cmDOjQlNSkBJ//hCISL8xsrnNuYHnXYrqlEE0s1L2UmBAgLaX61VpYHGTJpr18vX4X89ftZv763XywaAsA2S0a8cYdp5KemlTXYYuIHCZqWgpm1hP4CZAJTHPO/fNYz0RTSyEcdh8oYMbSbfz8lW84r2dL/nntAHUriUitVdZSCOsIp5k9bWZbzWzBEeXDzGypma0ws3sAnHOLnXM/Aq4ETgtnXLGiScNkRvZvy70X9uC9hZsZ/8lqv0MSkTgX7mkvE4BhZQvMLAF4ErgQ6AVcbWa9QtcuAd4FJoc5rphyy+mdubBPKx54bwlfrtrhdzgiEsfCmhScczOBnUcUnwyscM6tcs4VABOBEaH7JznnLgSuqegzzWy0mc0xsznbtm0LV+hRxcx46PK+dGzWkDEvfc3WPXl+hyQiccqPCfJtgfVl3ucAbc1sqJk9bmb/opKWgnNunHNuoHNuYFZWVrhjjRrpqUmMve5E9uUVceeL8ygsDvodkojEoahZNeWcm+Gc+7Fz7jbn3JN+xxONurdM54HLjmf2ml08OGWJ3+GISBzyIylsANqXed8uVFZl9fmM5hH92nLDKR0Z/+lqJn+3ye9wRCTO+JEUZgPdzKyzmSUDVwGTqvMB9f2M5v+9qBf9OzThl698w4qt+/wOR0TiSLinpL4EfA4cZ2Y5ZnaLc64IGANMBRYD/3XOLQxnHPEmOTHAP64ZQGpSAleN+5yPl9WPAXcRCb+oWbxWHWY2HBienZ196/Lly/0OxzfLt+zlzhfnsWzLPkYP6cIvzj+O5MSoGSYSkSjl2+K1cKnv3UclurVMZ9KY07lmUAfGzVzFFWM/Y+2O/X6HJSIxLCaTghySmpTA/d8/nn9eM4DV2/dz0eOf8ubX1Rq3FxEpFZNJoT7PPqrIhce3ZvJPzqBHq3T+5+X5PDNLW2KISPXFZFJQ91H52jVtyMTRgzmvV0v++O5iPlux3e+QRCTGxGRSkIolJgT465Un0DkzjTtfnMf6nQf8DklEYoiSQhxKT03i39cPpDjoGP38XA4UFPkdkojEiJhMChpTOLbOmWk8fnV/lm7ew92vfkssTj0WkciLyaSgMYWqGXpcC+4e1oN3vt3E3z5czo59+UoOIlIpHccZ524b0oVFG/fw2LTlPDZtOQ2SEmjXtAHdW6bzuxG9yWyU4neIIhJFlBTinJnxlytPYGT/NqzdcYCcXQfJ2XWADxZv4UBBEU/feJKO+BSRUkoK9UBSQoCze7Q8rGzCrNX89u1F/OeLtVx3Sid/AhORqBOTYwoaaK69G07txJnds/jju4tZvmWv3+GISJSIyaSggebaMzMevqIvjVIS+fHE+eQXFfsdkohEAXUf1WMt0lN56PK+3PLsHG57fi7Ht82gUUoijVITSU9NIj0lkewWjWjfrKHfoYpIhCgp1HPn9GzJj8/pxoRZq5m5bBvBI2asJicEmPrTIXTOTPMnQBGJqJg8T6HEwIED3Zw5c/wOI2445zhQUMy+/CL25hWybW8BN034iu8d35q/XtnP7/BEpI7E3XkKEh5mRlpKIi0bp5LdIp1Tujbn2kEdeWv+RtZs1zkNIvVBTCYFzT6KnNFndiExYDwxfYXfoYhIBMRkUtDso8hpkZ7KtYM78sbXG3Sqm0g9EJNJQSLrtlBr4Um1FkTinpKCHFOL9FSuGNiON+dvZOf+Ar/DEZEwUlKQKrn+lE4UFAV5efZ6v0MRkTBSUpAq6d4yncFdmvGfL9ZSfORiBhGJG0oKUmXXn9KJDbsPMn3JVr9DEZEwicmkoCmp/jivV0taNk7huS/W+h2KiIRJTCYFTUn1R1JCgFEnd2Tmsm3c/+4iNuw+6HdIIlLHYjIpiH9uOr0Tl5zQhqdnrWHIQ9N5/vM1fockInVISUGqpXFqEo9f3Z+Zd5/FGd0y+e3bi/hsxXa/wxKROqKkIDXStkkDnhg1gC6Zadz54jzW7zzgd0giUgeUFKTGGqUkMu76gRQFHaOfn8vBAh3UIxLrlBSkVjpnpvH41f1ZsnkPd7/2LbG8FbuI6JAdqQNnHdeCX15wHA+9t5Si4iAj+rXl7B4tSE7U7xwisUZJQerE7Wd2JfdgIRO/Ws+UBZsZ0KEJY687kRbpqX6HJiLVoJPXpE4VFQd559tN3PP6tzRtmMy1gztyyQltdM6zSBTRyWsSMYkJAUb2b8urPzqVFo1TeXjqUi7752faL0kkRsRkUtA2F9GvT9sM3rrzNP565Qls3ZvPgg36fyUSC2IyKWibi9hxRrcsAGat1AI3kVgQk0lBYkdWego9WqXz2YodfociIlWgpCBhd2rXTGav2UleoRa3iUS72J6SunM1vDSqZs+mZkDXsyH7HGjYrG7jksOclt2cp2etZt7aXZyanel3OCJSidhOCsX5sHtdzZ7duxG+eREsAO0HQbfzvVfL3mBWt3HWc4O6NCc5McD/vbWAh684gQEdmvodkohUoP6uUwgGYeM8WDYVlk+FTd945Y3bQffzodsF0HkIJGt+fV34dPl27n71G7bvL2Daz87UugURH1W2TqH+JoUj7dkEKz7wksSqGVCwDxJTodMZ0P0CrxXRtGPdfK96alPuQYY+PIPvHd+aR3/Qz+9wROqtypJCbHcf1aXGrWHA9d6rKB/WfgbL3/eSxORfePdk9fCSQ/cLvC6nhCR/Y44xrTMacNNpnfnXzJWkpybSNasR3x/QlsapqkeRaKGWQlVsX+F1MS2b6iWLYCGkZED22V43U7fzIE0DqFWRe6CQa576gtXb9rO/oJj0lEQevLwv3zu+td+hidQb6j6qS/l7YeV0L0ks/wD2bQEM2p7otSD6XAbNu0Y2phj1XU4uv5m0gK/X7WZAhyaMHtKFYX2UHETCTUkhXIJB2PwNLHvfSxIb5gHOG4cYcAP0HA5J2iW0MgVFQZ6ZtZpX5uawYus+BndpRpesRgQMmjRI5vIT29EpM83vMEXiipJCpOzZCPNfhHnPwe61kNoETrjKG6do2dvv6KJaYXGQf328kikLNrM5Nw+A3QcLCTrHBb1a8dAVfTX2IFJHlBQiLRiENTO95LD4bSgu8LqXBtwAfS6FlHS/I4wJW/bk8Z8v1jL245U0Tk2iQ/OGdM5Mo2OzNK4/pSNN05L9DlEkJsVMUjCzkcBFQGPgKefc+5XdH7VJoaz9O+Dbl2Hes7BtCSQ38hLDgBu8RKGFcsf02YrtvDxnPRt2HWTj7oNs2pPH0O5ZPHPTyX6HJhKTfE0KZvY0cDGw1TnXp0z5MOAxIAEY75x7oMy1psAjzrlbKvvsmEgKJZyDnNkw91lY+DoUHoAWvb3B6YRj/MbbKAs6DYHMbkoiwL9nruL+yYs5PTuTe7/Xg95ttFuuSHX4nRSGAPuA50qSgpklAMuA84AcYDZwtXNuUej6X4AXnHPzKvvsmEoKZeXtgQWvea2HjV9X/bn01t4q685DoPOZ0KR9+GKMYoXFQf4xfSXPf7GWoHN8ce85Og9apBp8XbzmnJtpZp2OKD4ZWOGcWwVgZhOBEWa2GHgAmFJRQjCz0cBogA4dOoQr7PBKbQwDb/Jex+Ic7FoNq2fCqo9hxTSvOwqgaWevC6p5V2jWBZqFvjZsFtctiqSEAD85txu92jTm1ufm8PmqHZzZPcvvsETigl8rmtsC68u8zwEGAXcB5wIZZpbtnBt75IPOuXHAOPBaChGI1V9moR/4XeDEG70ksXWxlyRWz4Scr7zuKBc89EzD5nDu72DAdb6FHQlndMskLTmBP727mDXb99O9ZTqDOjcjEIjfhCgSblG1zYVz7nHgcb/jiGpm0LKX9xr8I6+sKB92rYWdq2DnSlgyGSaN8bqmhj0AifE5Syc1KYE/X9aXRz9Yxn2TFgJwZvcsfj+iNx2ba22DSE34lRQ2AGU7xNuFyqrEzIYDw7Ozs+s6rtiUmAJZ3b0XwMm3wbTfwWePw9ZFcMWzkN7S3xjD5JIT2jC8b2tydh3k/UVbeHDKEs56ZAbD+rTigt6tyEpP4aROzUhK0JiDSFVEZEpqaEzhnTIDzYl4A83n4CWD2cAo59zC6nxuzA40R8qC1+CtMd4iuh88D+3KHVeKK1v35PHMZ2v4zxdr2ZtXBEBacgL/vPZEhmjcQQTwf/bRS8BQIBPYAtznnHvKzL4H/A1vSurTzrn7q/vZSgpVsPk7mHgN5OZA3yvh9J9C1nF+RxV2BwqKyNl1kIUbc/nbh8vZn1/ER78YqlXRIsTQ4rWqKtN9dOvy5cv9Dif6HdgJMx+GuROg8CD0vBjO+Dm06e93ZBHxXU4uI/8xi7N7tOBf156ogWip9+IuKZRQS6Ga9m+HL8fCl+MgPxeOvwLO+z00buN3ZGH3zKzV/O7tRXRq3pDebTP4zcW9aNlYmxVK/aSkIIfL2+MNQs96HAKJMOTncMoYb8A6TjnneOHLdXy4eAtfrtpJalKAn5zTjYGdmtGnrVZES/0Sd0lB3Ud1ZOdqeP/XsOQdaNTKOwvi+MugzYC4Xvy2Yus+fvzS1yzatIeAwSs/OoUTOzbzOyyRiKlVUjCzh4A/AgeB94C+wE+dc/+p60CrSy2FOrJyOswe7x0/WlwATTtB9rnQZai3pUZq/P0mXRx0LN+6lyvHfs5ZPVrw2FX1Y3xFBCpPClWZvH2+c24P3qZ2a4Bs4Jd1F574rutZcNUL8IvlcMkTkHkczH8JXr4W/tIDPn0Uigv9jrJOJQSMHq0aM6xPKz5aspWi4uCxHxKpB6qSFEoWuF0EvOKcyw1jPOKnBk28rTGu+S/8ag3cNAW6ng0f/hbGng5Lp8Rdcji7Rwv25hXxyPvLiMWuVJG6VpXuoweAkXjdRycDTfAWog0Kf3gVxqQxhUhaOgUm3w2567x9lbpf6HUpmUGn06Hb+RBI8DvKGgkGHf/31gJe+HIdnZo35I07TtPhPRL3aj3QbGbNgFznXLGZpQHpzrnNdRxntWlMIYKKCmDlNPjuFVj5ERQXQbAQivKgSQe48GE4bpjfUdaIc477Ji3kuc/XAvDRz8+kS1Yjn6MSCZ/aDjRfWtl159zrtYitVpQUfFZcCEvehRkPeCum7/jMSxAxqGTK6q/fXECXzDTG3zBQiUHiVm0Hmm8BngKuCb3GAzcDw/EGn6W+SkiC3iNh1ETAwZt3eOdTxyAz49rBHXnsqn6s2bGf2/8zj4MFxX6HJRJxVdklNQno5ZzbBGBmrYEJzrkqnBAj9ULTTt4W3ZPGwKs3Qq8R3ilxianeVhoxtOZhRL+2NG2YzA3PfMW9r3/LqEEd6di8oVY/S71RlaTQviQhhGwBfO0j0NbZUaj/tbBtCcx7Hha9dai8y1kw4gnIaOdfbNU0pHsWd52VzeMfreDN+RsBePjyvlwxsH4efyr1S1XGFJ4AugEvhYp+gHeU5l1hju2YNKYQhYqLYPM3kL8XtiyCj/7ozUy64H6vBZGYGhPbaTjnmL9+N+t2HuBPkxezZU8+p3Rpzi8u6K7VzxLz6mL20feBIaG3M51zb9RhfDWmpBADdq3xznRY84n3PrGBNw7RrCs06+wlioTo3s56f34RY16cx/Sl2wCYNOY0+rZr4nNUIjVXF0mhI9DNOfehmTUEEpxze+s4zmpTUogRwSAsehP2boLty+C716Ag9NencTvvWNHM46D9yd4Cuii1bscBzv3rxxQUB/njyD5cO7ij3yGJ1Ehtp6TeCowGmjnnuppZN2Csc+6cug+1epQUYlQwCK7Y23Np1mOw9lOvvHk3uP5NaNAUElIgIaqOEAdgxtKt3PjMbAA+/NmZZLfQtFWJPbVNCvPxVjJ/6ZzrHyr7zjl3fJ1HWk1KCnFi+wrYuhBevw2KDnplGe3h5veicoB68aY9XPiY1x32lytO4LIToy9GkcrUdp1CvnOuoMyHJQK+bhJjZsPNbFxurrZhiguZ2d7Yws1TvEN/zrkPDu6Gpy+E10fDno1+R3iYnq0b88yNJ9EgKYGfv/INa7bv9zskkTpT1a2zdwPXA3cBdwCLnHP/G/7wKqeWQhxb+RHMfAQ2fg1JDWDkWOh+vt9RHWbqws3c9vxcAE7t2pw7z8rmtOxMn6MSObbadh8F8FY1nw8YMBUY76JgS0klhXpg2zJ49SbYsgBO/bHXioiisYb3F27mt5MWsjE3D4BzerTgwcv7ktko+qfdSv1V46RgZgnAc865a8IVXG0oKdQThQdh6v+DOU9Dw0zvhLhT7vRWSjfMhOSGvoZXUBTkuw27+cnE+eTs8sZEbjm9M/93cS9f4xKpSG1bCp8CZ5cdV4gWSgr1zHevwsI3vONDS6Rlwbm/gz6Xet1MPhv/ySoemLKEoqDj8hPb8dtLetMoJXpaNiJQ+6TwHNATmASUjqg55/5al0HWhJJCPbV6JuxeB8Eib4fWvZu87TSueTUqupa27snj5D9NA6BvuwzeuvM0LIb2f5L4V6PZR2b2fOg/LwHeCd2bXuYl4o/OQ7y9lk68EcbMgbN+Daumw9LJfkcGQIvGqcz/zXm0SE/h25xcnpy+wu+QRKqswpaCmS0CzgXeA4Yeed05tzOskVVCJ6/JYYLF8NgJ0Litd4RooCozrcNv94EC+v3+AwAW/u4C0tSNJFGipusUxgLTgO7AnDKvuaGvvnHOve2cG52RkeFnGBItAgkw5Jew/gt4qDM81AWeHAQ7VvoaVpOGyfx+RG8Aet83lQUbtK5Gol9VxhT+6Zy7PULxVIvGFKSUczDnKdi62Hv/3SuQtwda9PRaDz7tqVRYHORPkxfzzKw1ALw95nSOb6dfZsRftd4QL1opKUiFNsyDha/DZ09401cvuN/XcJ7+dDW/f2cRAL84vztjzu7mazxSv1WWFNTJKfGp7QDvdXA3fDkWTrzJ207DJzef3pnmjZJ5eOpSHnl/Gad3y6Jf++jdEVbqr+gYkRMJl3N+453h8OTJ8Ics+Pc53sC0D0b0a8v93/f2kRz55Cy27snzJQ6RyigpSHxr1AKufRVO+zEcfwVsmAN/7QV/7Q0vX+uNRUTQmd2zuC50DsPgP09j2978iH5/kWNR95HEvw6DvVcwCOmtYO8W2LsRFr8Nz1wIgUSwAAy9BzqeGvZw7hveizU79vPJ8u3c8PRXnNixKX8Y2Sfs31ekKjTQLPVTwX5vW+4DoeU2WxZ46xy6nu29T28Fp97l7a8UBnmFxfzPxPks2byHNTsOcNuQLowe0oXm2khPIkCzj0SO5fMnvS0znPO2zyg6CBf9xetySg3fFNKv1+3i5gmz2XWgkOsGd+SC3q04tWtzAgFtiyHhE3dJQSuaJaz274BHe0FRHpz0Qy85hFEw6Oj/hw/IPVgIwIs/HMSpOpdBwqi2J69FHa1olrBKaw53zYUuQ2H2eO+wnzAKBIwPfjqEZ28+GYBR47/kZy/PD+v3FKlITCYFkbDLaAfDHvT++6tx8NYYmHw35IVnq4oWjVM5s3sWfxzZh56tG/P61xt0zKf4QklBpCItesCl//ZmJy1/H776F3z9H29PpaLwTCW9dnBHfnh6ZwCe+3wtRcXBsHwfkYrE5JhCCQ00S8QEi+HBzpAfain0GglXPhu2b3fS/R+ybW8+N5/Wmd8M1wluUre0zYVIbQUS4Ia3YPtymPccrJwOb/+Pdy2jHZzx8zqdvvrMjSdx+wtzeX/RZvKKijmlS3OGn9Cmzj5fpCLqPhKpqjb9oe+VMOg271zoJe/Cgtfhoz9A7vo6/VZ92mYwvG8b8gqDvDY3h0feX1qnny9SEXUfidTGmlkw4XuQ1NAbe2jcBkZ/DEmpdfYt7ntrAc9+vpbWGam8ccdptMqou8+W+kndRyLh0n4QDL3Xm5W0czUsmwIrp0HTTt71xFRo1qVWXUvXndKRXQcKmfTNRqYv3cqIfm1omKx/uhIeaimI1JVN38K/zji6/Pq3vDUPtbB6+37OemQG4G2qV7KmQaQm1FIQiYRWx3sJoGQtQ/4+eOsOb+yhuMgrS2nktS6q2XLonJnGi7cO4vFpy1m8aQ8zlm4lJTGBkzo1JTFBQ4NSd9RSEAmXYBAe6nT0grdbp3sHANXA36ct5y8fLCt9P/baAQzr07oWQUp9pJaCiB8CAbj9M9izyXu/azW8fiusmg7FBZDSGFpWbw3CbWd25YzuWRzIL2LU+C/5YtVO+rTNoF3ThmH4A0h9pKQgEk4Z7bwXQPOu3rkN035/6PodX3orp6soOTFAv/ZNcM6R0SCJCZ+t4bV5OXx73/lYmLb5lvpFnZEikdKwGdz2CVz3xqF9lXLXe9t1l7yqyMx4887TuG5wR/bmFZF7sBDnHLHcHSzRIWrGFMysC/C/QIZz7vKqPKMxBYlZO1bC348YV2jZB26fVa2PeWv+Bn4y8dCOquf2bMH4G06qiwgljvk2pmBmTwMXA1udc33KlA8DHgMSgPHOuQecc6uAW8zs1XDGJBIVmnWB7z0CB3Z479d+Bqs/huJCSEiq8sec07MlvxrWg/yiYj5aspVvcsKzi6vUH+EeU5gAPAE8V1JgZgnAk8B5QA4w28wmOecWhTkWkehhBiffeuj9l//yksLkX0JimSM5u19w6IjQcjRKSeT2oV0ByCsMMv6TVfx20kIAerVpzJUD24clfIlfYU0KzrmZZtbpiOKTgRWhlgFmNhEYAVQpKZjZaGA0QIcOHeosVhFfte4HaVmw8PVDZfn7YOP8SpNCWQM6NCE9NZHX5+WQVxQkMWBKClJtfsw+aguU3T0sBxhkZs2B+4H+Znavc+7P5T3snBsHjANvTCHcwYpERIdB8MsVh5e9NAp2r6vyR5zfuxXn924FwGMfLufRD5dRWBwkSYvbpBqiZkqqc24H8CO/4xCJGslpsG0x/K3v4eUtesGoiZU+mp7q/dMe+vAMAgG4oFcrfn2xzmWQY/MjKWwAyrZp24XKqszMhgPDs7Oz6zIukegy8CZvXUNZ2xZ7m+4V5R8+9nCE83q1ZOnmvRQWB5m9dicfLdmqpCBV4kdSmA10M7POeMngKmBUdT7AOfc28PbAgQNvPebNIrGq46neq6zZ4+Hdn8PB3ZDessJH2zdryIOXey2Me177lulLt4YzUokj4Z6S+hIwFMg0sxzgPufcU2Y2BpiKNyX1aefcwnDGIRI3GjT1vo4/FxLK/PO1AJz7O+h58VGPNExOZOvefIY+PL20rFebxvzjmhPDHa3EoHDPPrq6gvLJwOSafq66j6Te6nwm9L8OivIOL1/0FqyeWW5S+H7/tuw6UEAwtFB1yaa9TFmwGeectsaQo0TNQHN1qPtI6q20TBjxxNHl676A/L3lPnJ8uwwe/UG/0vdPTl/Bw1OXUlAcJCUxIVyRSoyKyaQgIkdIbgSbv4PP/3H0tdZ9odPppW9TEr3B66c+XX1YUhjcpRm922SEPVSJbjGZFNR9JHKEzG6weBJMvffoaxnt4acLSt92ap4GwEPvLT3stsFdmjFx9ClhDVOiX9RsiFcT2hBPJCQYhPw9R5d/+FtY+Abcs/aw4v35RRQFD/3bv+ulr9m5P5937irnOFGJOzpkRyTeBQLQoMnR5akZUHjwqOK0lMP/6aenJrJh14FwRScxRElBJJ4lNYDifJjxAFDOTKMGTeGkH9IgKYGte/N57MPlpZdSkgKMGtSBxqlV37VVYl9MJgWNKYhUUVYPbw3DjHK3EvN0HkLP1o15dW4Oj3647LBLrTNSGdGvbZiDlGgSk0lBU1JFqqj3SOh5SfnXlk2BiaOg6CC3nN6Dm07tVHpp0548TnvgIw4WFEcmTokaMZkURKQaAhXsklqyd1JxYei2Q91LqaFpq/lFwbCGJtFHe+qK1FcJyd7X4oKjLqUkeesX8ovUUqhv1FIQqa9KksKbt3uL38po6BzvJe/jgxlDuGDuVYddS0kK8Lcf9KNL1uHPSHyIyaSggWaROtDqeOh3LeQffa5zAGi/+1MuCnzNwsxbSsv3FxTxyfLtfLchV0khTsVkUtBAs0gdSE6DkU9WeDnthSvpsm8zY687tJvqmu37GfrIDIqKY3fRq1ROYwoiUr6EpNJB6BKJCd5gdFFQA9DxSklBRMqXmHLUIHTJec+FainErZjsPhKRCEhI9rbIyM0pLUreX0BrdpC/Yx2b15dtRRjBRq3BjFaNUw+b3iqxRUlBRMqXnAZ7NsCjvRe09FsAAAoqSURBVEuLmgKfp+Idqjv78NsfK7qUR4su586zuvLLC3pEMlKpQzGZFDT7SCQCzvgFtO4HHN5VtGzLXnYfOHysod/iRxjeHiZsSGJzbn4Eg5S6FpNJQbOPRCKgcWsYcN1Rxd3Lu/fR8XRplkrjnUkUaxA6pmmgWURqL5AIwSISAnbYOQ0Se5QURKT2AokQLCQxYBQrKcQ0JQURqb3SlkJALYUYp6QgIrWXkAjBYrUU4kBMDjSLSJQJJMK6z/lz0XoO7ihm4Z8SKr09KSFAdotGBDBIbQyXjoOU9AgFK5WJyZaCmQ03s3G5uUdv5CUiPjhhFLToTWajFBomV54QCosdO/cXUFAUhIM7Yelk2L6s0mckcsy52G3qDRw40M2ZM8fvMESkGl6fl8PP/vsNH/9yKB13fgYvXA63fAjtT/I7tHrDzOY65waWdy0mWwoiErsSQltgFAedd340gNNhPtFCSUFEIipgXlIIOgeBUFdTUEkhWigpiEhEHWopABZKCmopRA0lBRGJqJKWQlEwqJZCFFJSEJGISgy1FIJqKUQlJQURiajS7qPDxhS0iV60UFIQkYgKaPZRVNOKZhGJqITQmMK0xVvYmbSN84Dln09i16IVlT6XGDB6t21MSkLli+OOkt4Sss+tYbT1T0wmBR2yIxK7stJTMIN/zFhJK3ZwVkqAbmterNrDX9fwm969Gho2q+HD9YtWNItIxO3Yl8+BAq/LKHBwJ1a4v9L7cw8WcOtzc/nZed24tH+7qn+j716Bj/4AP1viHRokQOUrmmOypSAisa15oxSal75reMz7U/blk+PWsa9BW2jaserfKC0z9B+x+8tvpGmgWUSinpWsgq72ttzeczjNbqoqJQURiXqBkp/t1X2wdHaTkkJVKSmISNQrbSlUu6FQkhTUfVRVSgoiEvWspKVQ3R/uailUm5KCiES9kv2Sqv0Lv2lMobqUFEQk6oV+tHvbbVfrQXUfVZeSgohEvUBtxxQ0JbXKlBREJOqV9gLV9Ie7uo+qTElBRKLeoYHm6j6ogebqUlIQkagXqOniNY0pVJuSgohEvdLZR9V9ULOPqk1JQUSiXu1nHykpVFXUbIhnZmnAP4ACYIZz7gWfQxKRKFHyC79mH4VfWFsKZva0mW01swVHlA8zs6VmtsLM7gkVXwq86py7FbgknHGJSGwxMy8xqKUQduFuKUwAngCeKykwswTgSeA8IAeYbWaTgHbAd6HbdDafiBzGgKkLt5Cz62CVn+m1dzU/BBa/8Cv2JWZUem92ViOaNkyuXZCRNPAm6DC4zj82rEnBOTfTzDodUXwysMI5twrAzCYCI/ASRDtgPpW0YMxsNDAaoEOHDnUftIhEpXN6tmTJ5j3MXruzys/kBJtwlnUg48AaKk8J0KA4GRJjaJi158Vh+Vg/xhTaAuvLvM8BBgGPA0+Y2UXA2xU97JwbB4wD7+S1MMYpIlHk39eXe1BYFYyq0zjiXdQMNDvn9gM3+R2HiEh95kdbaQPQvsz7dqGyKjOz4WY2Ljc3t04DExGp7/xICrOBbmbW2cySgauASdX5AOfc28650RkZx+olFBGR6gj3lNSXgM+B48wsx8xucc4VAWOAqcBi4L/OuYXhjENERKom3LOPrq6gfDIwuaafa2bDgeHZ2dk1/QgRESlHDM2/OkTdRyIi4RGTSUFERMIjJpOCZh+JiISHuRjeZ9zMtgFryxRlALlVfJ8JbA9DWEd+z7q4v7J7KrpWnbo48n19r5sjy+pz3ZRXVlHdQHjqR3VTserWTckzTZxzWeVedc7FzQsYV9X3wJxIxFAX91d2T0XXqlMXqpuj7zuiPupt3VTh78qR1+q8flQ3dVc3VXkmJruPKnHk9hjHeh+JGOri/sruqehadetCdVNxWX2um/LKVDcVl0V73RzzmZjuPqoNM5vjnKvpZipxTXVTMdVN5VQ/FYuVuom3lkJ1jPM7gCimuqmY6qZyqp+KxUTd1NuWgoiIHK0+txREROQISgoiIlJKSUFEREopKYSYWZqZPWtm/zaza/yOJ5qYWRcze8rMXvU7lmhjZiNDf2deNrPz/Y4nmphZTzMba2avmtntfscTbUI/c+aYWXjO1ayhuE4KZva0mW01swVHlA8zs6VmtsLM7gkVXwq86py7Fbgk4sFGWHXqxjm3yjl3iz+RRl416+bN0N+ZHwE/8CPeSKpm3Sx2zv0IuBI4zY94I6maP28AfgX8N7JRHltcJwVgAjCsbIGZJQBPAhcCvYCrzawX3glwJWdHF0cwRr9MoOp1U99MoPp18+vQ9Xg3gWrUjZldArxLLbbKjyETqGLdmNl5wCJga6SDPJa4TgrOuZnAziOKTwZWhH77LQAmAiOAHLzEAHFeL1DtuqlXqlM35nkQmOKcmxfpWCOtun9vnHOTnHMXAnHfJVvNuhkKDAZGAbeaWdT8zAnrITtRqi2HWgTgJYNBwOPAE2Z2EZFZnh6Nyq0bM2sO3A/0N7N7nXN/9iU6f1X09+Yu4Fwgw8yynXNj/QjOZxX9vRmK1y2bQv1oKZSn3Lpxzo0BMLMbge3OuaAPsZWrPiaFcjnn9gM3+R1HNHLO7cDrM5cjOOcex/uFQo7gnJsBzPA5jKjmnJvgdwxHipomSwRtANqXed8uVCaqm8qobiqmuqlYzNVNfUwKs4FuZtbZzJKBq4BJPscULVQ3FVPdVEx1U7GYq5u4Tgpm9hLwOXCcmeWY2S3OuSJgDDAVWAz81zm30M84/aC6qZjqpmKqm4rFS91oQzwRESkV1y0FERGpHiUFEREppaQgIiKllBRERKSUkoKIiJRSUhARkVJKCiLlMLMmZnbHMe4ZambvRComkUhQUhApXxOg0qQgEo+UFETK9wDQ1czmm9nDodcCM/vOzI46TMfMTjKzr82sa+hErafN7KtQ2YjQPTea2etm9p6ZLTezh0LlCWY2oczn/zTCf1aRUtolVaR89wB9nHP9zOwyvF1iTwAygdlmNrPkRjM7Ffg7MMI5t87M/gR85Jy72cyaAF+Z2Yeh2/sB/YF8YKmZ/R1oAbR1zvUJfV6TCP0ZRY6iloLIsZ0OvOScK3bObQE+Bk4KXesJjAOGO+fWhcrOB+4xs/l4W0enAh1C16Y553Kdc3l4J291BFYBXczs72Y2DNgTiT+USHmUFERqZxOQh/fbfwkDLnPO9Qu9OjjnFoeu5Ze5rxhIdM7twmuFzMBrkYwPf9gi5VNSECnfXiA99N+fAD8I9f1nAUOAr0LXdgMXAX8OnTQG3o6Yd5mZAZhZ2YRxFDPLBALOudfwznoeUJd/EJHq0JiCSDmcczvMbJaZLQCmAN8C3wAOuNs5t9nMeoTu3WJmFwNTzOxm4A/A34BvQ2fvrgYuruTbtQWeKXNO773h+VOJHJu2zhYRkVLqPhIRkVJKCiIiUkpJQURESikpiIhIKSUFEREppaQgIiKllBRERKSUkoKIiJT6/0RLkdEoKVoXAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "digram_freqs = [freq for token, freq in digram_vocab.token_freqs]\n",
    "axes = plt.gca()\n",
    "axes.set_xlabel('tokens')\n",
    "axes.set_ylabel('freqs')\n",
    "axes.set_xscale('log')\n",
    "axes.set_yscale('log')\n",
    "axes.plot(freqs)\n",
    "axes.plot(digram_freqs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "trigram_tokens = [[triplet for triplet in zip(line[:-2], line[1:-1], line[2:])] for line in tokens]\n",
    "trigram_vocab = Vocab(trigram_tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7f54915ff438>]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3dd3zU9f3A8df77rKBhABhJIQVpiAblA0uRHBvrVUp1IGt2tZqa5cd7toqLoqUH6JYxbpQwDoQUUACyIxsAmFDIIzMy31+f3wvIYQQcsldvjfez8fjHuQ+9/1+752v8d732WKMQSmllAJw2B2AUkqp4KFJQSmlVDlNCkoppcppUlBKKVVOk4JSSqlymhSUUkqVc9kdQF00bdrUtG3b1u4wlFIqpCxfvvygMaZZVa+FdFJo27YtmZmZdoehlFIhRUSyz/RaSDYficg4EZmSl5dndyhKKRVWQjIpGGM+MsZMTExMtDsUpZQKKyGZFJRSSgWGJgWllFLlNCkopZQqp0lBKaVUOU0KSimlymlSUEopVU6TglJKqXKaFJRSSpXTpKCUUqpcSCYFXeZCKaUCIySTgi5zoZRSgRGSSUEppVRgaFJQSilVTpOCUkqpcpoUlFJKldOkoJRSqpwmBaWUUuU0KSillCqnSUEppVQ5l90B1MXm/ce5YvKi2p0sgsshOB2CUwSX8+TPTkfZcwdOAafDgcshOBwnzyk/1/uzwyFEOR3ERTmJjXISF+0g1uUkNtpJrMtJXLST2KiTr8e6nCTEOHE5NS8rpYJHSCcFl0NonBBdq3M9Bjweg9vjwe3xUOQ2lHoMbo/1b9nDfdrPnlOee4yhpNTUKoYYl4Nz0xLpk96Y3ulJ9ElvTEqj2FpdSyml/EGMqd0HWjDo16+fyczMtDsMwEowxaUeCktKKSzxUFBSSmFJafm/RaeVedh9pICVOw6zdtdRiks9AKQmxdGnTWP6eJNEj9REHA6x+bdTSoUTEVlujOlX1WshXVMIJg6HEOuwmoZ8VeQuZd3uo6zIPsyKHYdZti2Xj1btBuDKXq147oZeiGhiUEoFXtAkBRHpCvwcaAp8box52eaQ6k2My0mf9Mb0SW9cXrb7SAHTv93OlIVb6d8umVsGtrExQqVUpAhoL6eITBOR/SKytlL5aBHZICKbReRhAGNMljHmLuB6YHAg4woFrZLieHh0F4Z1asafPlrP2l26TLhSKvACPfRlOjC6YoGIOIEXgUuBbsBNItLN+9rlwMfAJwGOKyQ4HMJz1/ckOT6aSW+u4Fhhid0hKaXCXECTgjFmIZBbqXgAsNkYs9UYUwy8BVzhPf5DY8ylwC1nuqaITBSRTBHJPHDgQKBCDxpNGsTwws292Xm4gIf/u4ZQHhiglAp+dgySTwV2VnieA6SKyAgReV5EXqWamoIxZooxpp8xpl+zZs0CHWtQ6N82mV9e3JmPV+9h5pJsu8NRSoWxoOloNsYsABbYHEbQ+umw9ny37RB/npNFr9aN6ZGmu84ppfzPjprCLqB1hedp3rIai8Q9mh0O4dnre9GkQTT3vrmCo9q/oJQKADuSwjKgo4i0E5Fo4EbgQ18uEKl7NCcnRDP55t7sOlLAr2ev1v4FpZTfBXpI6ixgMdBZRHJEZLwxxg1MAuYDWcDbxph1gYwjnPRtk8xDl3Rm7tq9THpzpdYYlFJ+FdA+BWPMTWco/4Q6DDsVkXHAuIyMjNpeIqRNHNYeAzw9fwNrd+fx4s196J4aWbUmpVRghOQSnZHafFRGRLhreAf+M/E8iko8XP3St7y+eLs2Jyml6iwkk4Ky9GubzCc/H8qgjCb87oN1TJq1Uie4KaXqJCSTQiSOPjqT5IRopv24P78e3YV5a/dy69SlFJaU2h2WUipEhWRSiPTmo8ocDuHuER148eY+rMrJ448far+9Uqp2QjIpqKqN7t6Ce0d24K1lO3nrux12h6OUCkGaFMLMgxd1ZmjHpvz+w3WszjlidzhKqRATkklB+xTOzOkQ/nljb5o1iOHumSvIPVFsd0hKqRASkklB+xSql5wQzcu39uHA8SJ+Nmslu48UUOrR4apKqbMLmgXxlH+dm5bEX67ozkPvrmbQE18Q5RRSk+Jo36wBj17WlfbNGtgdolIqCGlSCGPX929NRvMGZO05ys7cAnYezufrjQe4b9ZK3rtnMNGukKwoKqUCSJNCmKu89/P8dXv56evLef7zTfzyks42RqaUCkYh+VVRO5pr75JzWnBt3zReWrCZFTsO2x2OUirIhGRS0I7muvnDuG60TIzjwf98T36x2+5wlFJBRJuPIlDD2Cievb4nN/1rCRNnLKdX6yQaxrpoGBvl/dfFOa0SadYwxu5QlVL1TJNChDqvfRN+eXFnXv1qC99uOUjlEatNEqL56qGRNIjRPxGlIon+Hx/B7h2Zwb0jMzDGcKK4lGOFJRwrdLNh7zHum7WSmUuyuWt4B7vDVErVo5DsU1D+JSI0iHHRMjGOTs0bMq5nK4Z2bMrUr7dSUKwrrioVSUIyKejoo8CbNDKDg8eLeWuZLqynVCQJyaSgo48Cb2D7Jgxom8yUhVspcmttQalIEZJJQdWPSaMy2JNXyHsrdtkdilKqnmhSUGc0tGNTurZsxP8tztb9n5WKEJoU1BmJCLcMTCdrz1G+36l7MygVCTQpqGpd2TuVhGgnbyzVDmelIoEmBVWtBjEuruidykerdpOXX2J3OEqpAAvJpKBDUuvXLQPTKXJ7eHdFjt2hKKUCLCSTgg5JrV/ntEqkd3oSLy3YwptLd1BYokNUlQpXIZkUVP378xXdaZUUy2/eW8Pwp79k+8ETdoeklAoATQqqRrqnJvLBvYN58ycDKXJ7uGvmcl0CQ6kwpElB1ZiIMCijKf+4oRcb9h3jN++t0fkLSoUZTQrKZyM6p/DAhZ14b+UuZi7JtjscpZQfaVJQtTJpZAajuqTw2Jz1uq2nUmFEk4KqFYdDeO76XrRMjGPijOW8+OVmdubm2x2WUqqONCmoWkuMj2LKbX1JaxzH0/M3cMHfv+KjVbvtDkspVQcSyh2F/fr1M5mZmXaHoYCcw/k88J/vWbb9MNf2TePKXqkM6djU7rCUUlUQkeXGmH5VvaY1BeUXaY3jmfmTgdx2fhvmrtnDra8tZeHGA3aHpZTyUUgmBV3mIjjFuJw8dkV3lj16IbFRDr7csN/ukJRSPgrJpKDLXAS3+GgXfds0ZvGWQ3aHopTyUUgmBRX8zm/fhB/2HuPQ8SK7Q1FK+UCTggqI8ztYncxLt+XaHIlSyhcuuwOok9xtMOvm2p0blwQdRkHGhdbPyq/OTUskPtrJ4i2HGNOjpd3hKKVqKLSTQmkRHKnljmA7FsP3b4DDBW0GQadLofNoSG7v3xgjVJTTwfntm/Duihy6tWrEjf1bIyJ2h6WUOovInafgKYWcTNjwCWycBwd+sMqbdYFOo6HzpZDWHxxO/wUcYfbkFfCLt1fx7ZZD/P36nlzdJ83ukJRSVD9PIXKTQmW5W2HjfCtJZH8LHjfEN4GOl1g1iA6jIKahf94rgng8hnGTF5FXUMIXvxhBtEu7sZSymyYFXxUcgS2fw4Z5sOlTKDwCzmhoOwQ6j7FqEkmt/f++YWrBhv3c/u9ljD6nBT3SErm2bxrNG8XaHZZSEUuTQl2UumHnEtgw13rkbrHKm3f3NjONgVa9waHfgM/EGMNdM5fzzeZDnCh243IIj1zalTsGt9V+BqVsoEnBnw5uspLDxnlWZ7XxQIPm0PFiOOdKaD9KE0Q1sg+d4M9zsvgsax/dWjbiziHtuKZPqiYHpeqRJoVAyc+FTf+DjXNh8+dQdBSS0qHPj6H3j6Bhc/tiC2Iej+GtZTt5Y2k263YfpV+bxnRq0RCHQJOEGG7o35pWSXF2h6lU2NKkUB/cxfDDHMicBtu/toa6drkM+t0JbYdp7aEKpR7DtEXb+GDVLvYcKQTgcH4xIsI1fVL585XdiXHp6C+l/E2TQn07uBmW/9uaB1Fw2Jr70Pd26HULJOhy0tXJOZzPa4u28e9vtpPSMIYOzRqQ2jiOnmmJ3DQgHZdTk6tSdRUySUFErgQuAxoBrxljPq3u+KBNCmVKCiHrQ6v2sGOxNYKp6+XQ7w5oMxi0Hf2MPlmzh7lr97Lt4HH2Hy1i/7Eifj26C3eP6GB3aEqFPFuTgohMA8YC+40x3SuUjwb+CTiBqcaYJyq81hh4xhgzvrprB31SqGh/FmT+G1a9BUV50LQTdBlrJYrqNO0I7YZDg2b1E2eQ+sn/ZbJw0wGu6ZPKw6O7khgfZXdISoUsu5PCMOA4MKMsKYiIE9gIXATkAMuAm4wx672vPwu8YYxZUd21QyoplCnOh3XvWbWHXT7E3rwHtB8O7UdCm/MhOiFwMQah/ccKeWb+Bv67YheX92zF32/oZXdISoWs6pJCwNc+MsYsFJG2lYoHAJuNMVsBROQt4AoRyQKeAOaeKSGIyERgIkB6enqgwg6c6HjofYv1OBtPKez5HrYugC1fwndTYPFkq3aR1h9SukKTDEjuAE06QFIbcIb2clZnktIwlqeu7QnA3DV7KXZ7dHa0UgFg1ydIKrCzwvMcYCBwH3AhkCgiGcaYVyqfaIyZAkwBq6ZQD7Hax+GE1L7WY+gvrFrGjsVWksj+Bla/bQ2DLT/eZTVJXTE5bJfkuOScFrydmcODb3/PyM4pnN+hiQ5fVcqPguprpTHmeeB5u+MIWtHxkHGB9QAwBk4chEObrZnWe1bDsqlwYAPcNAuS29kbbwAM79SM2we15Z3MncxZvYcop/DbMV25eWAbrTko5Qf1MvrI23w0p0KfwvnAH40xl3ifPwJgjHm8htcbB4zLyMiYsGnTpoDEHLK2fAnv3A7igOtnQLuhdkcUEAXFpWw7eILH52bx9aaDtGgUy+2D25KeHE+/No1J0bWVlDoj24ekVpEUXFgdzRcAu7A6mm82xqzz5boh2dFcHw5tgVk3Wiu/XvoU9K92EFdIM8awcNNBXvpy8ym7vF3dO5VnruuJw6HDfpWqzNaOZhGZBYwAmopIDvAHY8xrIjIJmI81JHWarwlBVaNJB/jJZzB7PHz8oDXaafhD0HZo2M2NEBGGd2rG8E7N2Jmbz76jhbyxdAf/XbmLQRlNubav7uGglC+CavJaTWnzUQ15SuG7f8Gi5+D4Xkg/H4b9ytobIsySQ0Uej+Hql79ly4HjfDRpCG2bRtbwXaXOxvbmo0DR5qMaKimEFTOs5HBsN3S4AMY8bdUowtTO3HzGTV6Ex2Po2rIRP7+wI4M66BIjSoEmBVXGXQTLXoMv/2btbz345zDkQWtUUxhatzuPyV9sZs2uPHYfKWDCsPYM6tCUYR2b6lLdKqKFXVLQ5qM6OrYXPn0U1rwDCSnQ4zo493po2TMsm5VOFLl59P21vLdyFwC/H9uNO4eE33BdpWqqTklBRJ4C/gIUAPOAc4EHjDEz/R2or7SmUEfbF8GSl629qT0l1uzojIuseRBtBoddDWJPXgF3vb6cvIISvvzlCK0tqIhVXVKoyWyfi40xR7EWtdsOZAC/8l94yjZth8CNb8AvN8LY5yCxtbUm0xvXwj+6w6r/WBPkwkTLxDiu7dea7Yfy2XrwhN3hKBWUapIUyoatXga8Y4zJC2A8yg7xydZmQLe9Dw9nwy3vQuN28N5EmHk17FoRNslheMdmiMDjn/xAsdtjdzhKBZ2aNB89AVyJ1Xw0AEjCmog2MPDhnTEm7VMINE+pVWv47E9QfMxa6rv9SGt9pZgGcO4NITt6acbi7fz+g3U0jHHx0X06ZFVFnjp3NItIMpBnjCkVkQSgoTFmr5/j9Jn2KdSDgiOw/n2rKWnvGqus5IRVc+h8KVz5MsQl2RtjLby+eDu/+8CaL/nOXefTv22yvQEpVY/q2tF8dXWvG2P+W4fY6kSTgk2O7YPM12DhM1aN4aqX7Y6oVuav28tPX19Oi0axvHZ7P85plWh3SErVi7p2NI8HXgNu8T6mAncC47A6n1WkadgcRv4GhjwAq960Ri+FoEvOacH0O/pzOL+Yu2Yu52hhid0hKWW7miSFKKCbMeYaY8w1wDlAlDHmDmPMnYENTwW14Q9BSjf4YBJ8/SxsWwjZ31pNTiFiROcU3pwwkN1HCvn17NV8ty2X/ccK7Q5LKdvUpPkoyxjTtcJzB7CuYll9047mILJvHbx/N+xZdbIsvilc/jx0ucy+uHz04pebeXr+hpPPb+7DZee2tDEipQKnrn0Kk4GOwCxv0Q1YW2ne59coa0H7FILIsb1w4AdrnaUv/2J1Sve6FS74vTVaKcj3lDbGsConj20Hj/PnOVnknihmZOdmPHxpVzq3CM9d7FTk8sfoo6uAYd6nC40x7/kxvlrTpBCk3MXw1ZOw6O9gvHMB2o+ENoMgoam1rEYQbxd6xNvHsGSrtT/D4kdG0TJRt/xU4cMfSaEN0NEY85mIxANOY8wxP8fpM00KQW73SquPIf8QrHoLjlprDxGbBAN/enL/6YTgXL10efZhrnn5WwBeHz+AoR2b2RyRUv5R1+ajCcBEINkY00FEOgKvGGMu8H+ovtGkEEKMsSbE7Vll1SB+mGOVN0qD2z+CBs3BFQsOp71xVvKfZTv49btrSE6I5uuHRpIQE1TbmitVK3VNCt9jzWReaozp7S1bY4zp4fdIfaRJIYQdzrb6IN79CRQdtcqS2sCd86FRcHXw/ndFDg++bXWkz7t/KF1aNLI5IqXqpq7zFIqMMcUVLuYCbF0IR0TGiciUvDxdhilkNW4DnS6xksBFj1kd0icOwLRL4MP7ID/37NeoJ1f1TuVPl58DwE9fX05BcanNESkVODVdOvsIcBtwH3APsN4Y89vAh1c9rSmEmY3zYdE/YFem1Zx03XRIq/LLjC1+894a3ly6A4A7B7fj1vPSad+sgc1RKeW7ujYfObBmNV8MCDAfmGqCYHceTQphatdyeOd2OLoHRj8OAybYHREA7lIPry/J5k8frS8v+82YLkwcFpoLA6rIVeukICJOYIYx5pZABVcXmhTCWMFheO8u2DjP2udh0M+g82hAoFEqOGrS8hkYR/KLmb08h798nAVASsMYXv1RX3qnN7YtJqV8UdeawiJgVMV+hWChSSHMeUphyUuQNQd2LjlZ3vo8uPCP0OZ8uyID4ODxIp6Y+wOzl+cAMPuu8+mnq62qEFDXpDAD6Ap8CJRvV2WM+bs/g6wNTQoRotRtDWEtPg55ObDgcav88snQ50f2xgY8++kGXvhiMwBvThjIoA7BOe9CqTK1Gn0kIq97f7wcmOM9tmGFh1L1w+mCc66E3rfCiIfh/rWQNgA++wO4i+yOjgcv6sSrP+oLwM3/Wkpevq62qkJXdQ2zfUWkFbADeKGKh210SGqES2oNIx+xZkqvftvuaBARLjmnBQ9c2AmAf3y+0eaIlKq9MzYficjPgLuBdsDuii8BxhjTPvDhVU+bjyKYxwPTLraW0oj1bo7T5zarr8G2kAxdfz+PIreHWwam89erbJ/fqVSVatV8ZIx53rs89r+NMe0rPNoFQ0JQEc7hgOtfh4F3wTlXQctesOg5+EtzWPmGTSEJM+4cgMshvLF0B4++vwaPx/aR20r5pEYL4gUrrSmocu4iyJxmLbx3eDv8bCXE2zMSaGduPqOeXUBJqaF1chxf/XIkDofYEotSVanrMhdKBT9XDJx3N1zxIhTmWftH26R1cjyfPjCcy3u2YmduAb9+d7VtsSjlq5CuKcS3izedH+tcq3NjnDE0i2tG84TmNI+3HinxKda/Cda/ybHJOETzZsj5YBKsfB2c0eCKgx9/CK161XsYx4vc9H7sU0pKDU9c3YMbB6TXewxKVaXO+ykEq/Ru6eYXM39Rq3ML3YXsz9/Pvvx97M/fz4H8A7iN+5RjXA6XlTjKEkZCpeQRn0JKfArRzmh//DrKXwqOwLJ/QfEJq0nJGIhpZHVI3/Y+NEipt1DW7spj7AuLAHjplj6M6RFcK8CqyBS2ScGffQoe4yG3MJd9+fvYd2LfKQlj34l9Vnn+PgrcBaedmxybTEp8CsPShnFvr3u1dhFM1n9oLbRnSmHVLGjW9eSmPn1vhx7XBjyEj1bt5r5ZK2nRKJZzWjXiyWvPpWmDmIC/r1JnoknBT4wxHC85fkqiKEse2Uez+W7vd1ze4XL+NOhPuBy6GUvQ+d8fYOd31s+5W0AccM7V1vP4ZBjyQMA2+Xnlqy18smYPq3PyuLhbc352QUe6pyYG5L2UOhtNCvXAGMOU1VOY/P1kLmpzEU8OfZIoZ5TdYakzyZoDH9xjzXcwpVCSDxf/1Zo1HZcUkLcsLCllzPNfs+3gCc5r14SJw9pzfocmxEYF125zKvyFXVIQkXHAuIyMjAmbNm2yO5xTzFg3g6czn2ZI6hCeG/Ecsa5Yu0NSZ1N0HJ7tbK2t1O9OGPtcQN/uzunL+OKH/QD8bmw3xg9pF9D3U6qysEsKZYKpplDR7I2zeWzxY/Rr0Y8XRr1AQlSC3SGpszm6G96/B7Z+ae0EN/jnAXur40Vuth88wU1TlnCsyE3fNo159+5BAXs/pSrTeQr17NpO1/L40MdZsW8FEz+dSF6RrtEU9Bq1gtFPWD8vecUa1vrpo+D2/4rxDWJcdE9N5OnrenJe+2SWZx9m5Y7Dfn8fpWpDk0KAXNb+Mp4d8SxZuVmMnz+eQwWH7A5JnU1KF2vym4i1uc+3L8D69+HQFqvvwc9Gd2/Bz0Z1BOCJuT9Q7Pb/eyjlK00KAXRB+gVMHjWZ7KPZ3DH/Dvad2Gd3SOpset8KD66HSctAnPDfCfBCH1j4VEDeblBGU3qmJbJ0Wy4PzV4VkPdQyhfap1APlu9bzr2f30tSTBJTL55KWsM0u0NSNZH9rXdTnyfA44YOo6zyDqOg2+V+e5vN+49z4d+/olnDGC7q1pyB7ZK5oleq366vVGXap2Czvs37MvXiqRwvOc6P5/2YrXlb7Q5J1USbQXDu9daS3CUF8MPH1gS4z/7g17fJSGnAb8Z0wRh4d3kOT83b4NfrK+ULrSnUo42HNzLx04kYDK9e9CpdkrvYHZLy1Rd/gYVPW8tmAPQf79c9HP48Zz2vLdpGq8RY3r93MCmNdEiz8r/qago67bYedWrciemjpzPhfxO4c/6dPDH0CVoktKj2nLQGacRHxddThOqs+twG7kLwlFrLZ/zwMfS47uTriWknN/2phZsGpJN7opj3Vu7if1n7uKZPmk5uU/VKawo22H18Nz/59CfsPLbzrMdGOaLo27wvQ1KHMCR1CO0T2yOia/MHhfm/hcWTTy1r1Qcmflmnyx44VsSAv32GMXBFr1b888bedbqeUpXp5LUglFeUx7K9yzCc+f6XmlLWHljLol2L2JK3BYCWCS0ZnDqYgS0H0iCqwVnfx+Vw0Selj67kGggFR2D712C8Q0m/fxO2L4Lr/u/kMWl9Ia6xz5fO3J7Ln+es51ihm9+P60bD2Cj6tvH9OkpVRZNCGNhzfA+Ldi/im13fsGTPEk6UnKjxuS0SWjDx3IlcmXElUQ5djylglk6Bub86taz3rdbch1p47KP1TPtmW/nzefcPpUuLRnWJUClAk0LYKSktYdORTZR4Ss567KGCQ0xbO41VB1aR2iCVCT0m0D7p7Ftst0poRfOE5v4IN3KUumHvaqu/AWDO/dZGP6Mft1ZkbdnT2iGuhgpLSsnac5TN+4/zq9mreeTSLlzXrzXJCVrrU3WjSSHCGWP4etfXvPj9i6w/tL5G58Q6Y/nb0L9xUZuLAhxdGJt9J6x99+Tzkb+F4Q/5fJmKfQwD2yXzn5+e78cgVSTS0UcRTkQYljaMoalDWX1w9VmbnowxvLTqJR5c8CD397mfO7rfgSDl11I1NOYZq/kIYPZ4yNtp7QJXpob3slnDGObcN4Qn5v7A1gMnKPsip/8tVCAETU1BRNoDvwUSjTE12g5LawqBU+gu5Hff/I552+eVl3Vs3JEpF02haVxTGyMLUS+dD/sr1NJiGllLaTSsfkhyRX/7JIspC09OfPzDuG7cMViX3Va+s62mICLTgLHAfmNM9wrlo4F/Ak5gqjHmCWPMVmC8iMwOZEyqZmJdsTw57EkGpw5mz/E9lHhKmJk1k3s+u4dpl0yjQfTZRz6pCi590lo2A+DwdmtmdO42n5LCj85rQ4MYFx5j+Pc321mzS1ffVf4X0JqCiAwDjgMzypKCiDiBjcBFQA6wDLjJGLPe+/psrSkEp0W7FnHf5/fRJbkLPVN6lpc3jWvKj7v9WHeaq6mcTJh6AXS+DJJanyxvOwS6jqvRJS7959ecKHIzqksKInB9v9Z0bakjk1TN2FZTMMYsFJG2lYoHAJu9NQNE5C3gCqBGPaAiMhGYCJCenu63WNXZDUkdwl+H/JWnM58me0u2VWjgWMkxikuLuafXPfYGGCqS20PjtpC9CLy3keJ82PJFjZPCoA5NeCdzJ/9dkcPRQjfFbg9/vapHwEJWkcOOjuZUoOJU3hxgoIg0Af4K9BaRR4wxj1d1sjFmCjAFrJpCoINVpxrTfgxj2o85peyRrx/hX6v/xYjWI+jWpJtNkYWQ+GT4eaVlsj+4FzZ/UeNL/G5sN3431rrXI57+kqOFbn9GqCJY0Iw+MsYcAu6yOw7lu4cHPMzSPUsZP388iTGnr/sT44zh2eHPktE4w4boQkR0Azi2B/5x7qnlbQbBVa9Ue2rD2Cj+t34vQ5/6gming3/e2JvuqbVff0lFNjuSwi6gQkMqad6yGhORccC4jAz9kAkGiTGJPD/qeWb9MIuq+qg+3/E5U1ZP4anhgdmoJiz0vBEKj55cMgNgzyrImgNXVX/qXcM78HnWPgrdpXyyZi+rc/I0KahaC/iQVG+fwpwKHc0urI7mC7CSwTLgZmPMOl+vrR3NoeHZzGeZsX4GH1/1sW4w5IuvnoIv/wq/Owg16MQ/VlhCjz9+yqOXdeUnQ88+a11FLjuHpM4CRgBNRSQH+IMx5jURmQTMxxqSOq02CUGFjlu73srMrJnc8sktNVrE70xS4lN49aJXI2dxv7KF9F7oC44Ky2dHxcP1M6BJh1MOj4+2/nee/OVmZtH59EwAAA+cSURBVC6xerCjnA6evq4nvVon1UvIKvQFevTRTWco/wT4pLbX1eaj0NI8oTm/Hfhblu1dVutr5BbmsmTPEtYfWk+vlF5+jC6IdboEdq+E0uKTZUXHYOM8q2mpUlJwOoRfXtyJTfuPA1BS6uGTNXv5fsdhTQqqxoKmo9kXxpiPgI/69es3we5YVM1c2+laru1Uo+knVTpYcJCRb49k5f6VkZMUktLhypdOLTuyw0oKxcerPGXSqI7lP+cXu/lkzV6K3J4qj1WqKiGZFFTkaRrXlNYNWzNv+zxcjpN/tkkxSYxtPzZy1gEqm0m+cT4UVUoMDhd0vwYSmgAQ47KanBZvPYTLeXI79tSkOEZ3r/lMahVZQjIpaPNRZBqeNpyZWTNPW+m1c3JnOjXuZFNU9SymETRoAT/MsR6VleTDkPsBqzkpPTmeBRsOsGDDgVMOW/unS2gQE5L/+6sAC8m/Cm0+ikwP9X+Iu3vdXf5865Gt/Gjuj9h8eHPkJAWnC+5fY334n8LAk+1Oa1b6/BfDyS8uLX/+TuZO/vJxFvlFbk0Kqkr6V6FChojQKPrk+j7dmnTDKU42H9lsY1Q2cEVbj8qi4qGk4NQip4PEuJNNR43jrfMKSkpRqiqaFFTIinZG07phaxbkLKjVYnwucXFVx6vCZynwqFjYsRgWPHn6a+KAc68nLtpKCq8t2kaTBGsXOIfA1X3TSE2Kq89oVZAKyaSgfQqqzODUwbyR9QabDm+q9TUmnBsmrZDNu8O2r2DX8qpfLzpK2+6/IsblYMbi7FNfcnv45SWd6yFIFexCMilon4Iq8/CAh3mov+9bXAIMnjWYQ4WH/ByRjW774NSd3Sp6JgOKT9CtVSOyHht9yks9/jhfm5NUuZBMCkpV5BDH2Q+qQnJsMrmFuX6OxkYiZ97i0xUH7kIAHI5Tj4l2OSjWuQzKq3b/NykVBsIuKVQnKva0TugyMS6nJgVVTmsKKmIlxybzze5vuOqDsyxDWoU4VxzPDH+GVg1aBSCyAHDFwab/WXtFV/JmyXEy13bikp0/P6U8NsrBP27sTbumCfUVpQoCIZkUtKNZ+cN1na+rVdPTsZJjLN2zlKxDWaGTFM67GzbOrfKlZsdWMqboO76o8OF/otjN15sOsnZXniaFCBOSSUE7mpU/DEkdwpDUIT6fl300m7HvjSXfXXkCWRDrfYv1qELDeb+BFf/HKz/qW162/eAJRjyzgJJSbVaKNNqnoJSP4lzWeP4Cd9Vt9CHHGXXqSqxAlMv6aNCkEHk0KSjlo/BLCtFWUqgwnDXau4CedkBHnpBsPlLKTrGuWMBaznvvib2+neuMJSk2yPY2KNu06Ei2tdIqEFPopiWHKDi0g7073SePdUThSUghyumgWcMYG4JVgaZJQSkfRTmiiHPFMX3ddKavm+7TuYLwzrh36JwcRLOHo70dyf/sWV7UCFgcC2R6HxVMKr6POZ7zmXpbPy7s1ry+olT1JCSTgo4+UnZ7+cKXyT6affYDK9h5bCdT10zlQMEBOhNESaH3LRCXBB73KcWb9h/n8ImTfQ1OTyF91z3OredEM2cN7DtWWN+RqnoQkklBRx8pu/Vt3pe+zfue/cAK1h1cx9Q1U3FX+vC1XWwi9Lr5tOKOlQuKT8C6x+nRMh7WQKnnDEtqqJCmHc1K1ZOyHeOCLinUlDd+p7E6n92lmhTCkSYFpepJlMNa3jt0k4IVv2Atnqc1hfCkSUGpelJWUyjxlNgcSS05HIDgNFZSc2tSCEuaFJSqJyHffATgcCGmrKagcxjCUUh2NCsVisqSwmtrX+PDLR/6/fqdGnfikYGP+P26p3C4cKx5m7ei5xH9tYN1S6r/XtkyMY5k7xagDJoEnS8NbHyqzkKypiAi40RkSl5ent2hKFVjTWKbMKbdGJrFNfP7tXcd38XbG972+3VP03880rgdzRvGEBNV/cfH8SI3uWVDWnO+g6w5gY9P1ZmYM+3UFAL69etnMjMzz36gUmFu8srJTFk9hdU/Xm13KOWGPvUFfdMb848be8NzPaDtYLjqFbvDUoCILDfG9KvqtZCsKSilTuUUJwaDxwRPO7/L4aB81KrDAR7d8jMUaFJQKgw4HU4ASk3wfPA6BDxlI5TECUEUmzozTQpKhYGyzYJKg+jbuNMhJ+cyOJxaUwgRmhSUCgMusUY2BVPzkUOEUqM1hVCjSUGpMFDWfOQ2wTMHwumQk81HDifovIaQoElBqTBQ1nzkCaIPXqejYk3BoTWFEKFJQakwUNZ8FEw1BYdon0Io0hnNSoUBh8P6fjd321waRje0ORprhzlxxLEnz807mTsZle/GdWw7G957odrzHAJdWzYiIdrHj6ZWvaF5tzpErMqEZFLQTXaUOlVKXAoATy17yuZITurW4H42Z7fgV7NX82pUDJc41zBg1aNnP/H7WrxZal+Y8EUtTlSV6YxmpcLE/vz9QbECa3ZeNj/97Kc8MeRpzm08zCosLcZ5Yt9Zz735X0u4oX9r7h7eoeZvOOcBOLYX7vm2lhFHnupmNIdkTUEpdbqU+BS7QwCgyF0EgNMBrZPjvaXx0CzprOfuka0ciWkJjdvU/A2jE4DQ/XIbbLSjWSnlVyIC1G7OhEMEnxsvRCCI5meEOk0KSim/Kh8ei+8f1FJxaYwan+TA90yizkSTglLKrxzej5Xa9Fc6RPB5QzdxaE3BjzQpKKX8qqz5yNSinV+kNudp85E/aVJQSvlV/fcpONCOZv/RpKCU8qu6NB+JgMfX87Sj2a80KSil/KquNQXfk4L2KfiTJgWllF/VZfSRQ2oxkEgc2nrkR5oUlFJ+VZYUatd8VIvRR9rR7FeaFJRSfiXUvvlIqEUy0T4Fv9KkoJTyq/Lmo/rsU9D2I7/RpKCU8qvy5qNafFA7hFpMXtOagj8FzYJ4IpIAvAQUAwuMMW/YHJJSqhbqMvpIajtPQZOC3wS0piAi00Rkv4isrVQ+WkQ2iMhmEXnYW3w1MNsYMwG4PJBxKaUCpy59Cg5HbfoUdO0jfwp0TWE6MBmYUVYgIk7gReAiIAdYJiIfAmnAGu9hum+fUiGqrPno460fsyF3g0/n5jfay4LDcPHr0TU+J70wi6YJkDtlcLXHRTkdJMZG+RSPbRq1hKTqlw+/rvN19E7p7fe3DmhSMMYsFJG2lYoHAJuNMVsBROQt4AqsBJGGte/SGWswIjIRmAiQnp7u/6CVUnUS54pjYMuB5BzLYcX+FT6d60ooobC4lH3FNT8nz1FEo5g4IK/a4wRwFIlP8djmaAEUH6r2kAvSLwjIW9vRp5AK7KzwPAcYCDwPTBaRy4CPznSyMWYKMAWsndcCGKdSqhYc4mDqxVPtDkPVUtB0NBtjTgB32B2HUkpFMjuGpO4CWld4nuYtqzERGSciU/Lyqq8uKqWU8o0dSWEZ0FFE2olINHAj8KEvFzDGfGSMmZiYmBiQAJVSKlIFekjqLGAx0FlEckRkvDHGDUwC5gNZwNvGmHWBjEMppVTNBHr00U1nKP8E+KS21xWRccC4jIyM2l5CKaVUFUJymQttPlJKqcAIyaSglFIqMEIyKejoI6WUCgypzUYYwUJEDgDZlYoTOXVqY8XnFX9uChz0c0iV39sfx1d3TFWv1aRM74neE70nkX1P2hhjmlV5hDEmrB7AlDM9r/RzZqDf2x/HV3dMVa/VpEzvid4TvSd6T870CMnmo7OovETGR9W8Fuj39sfx1R1T1Ws1KdN7ovdE70ntysL5ngAh3nxUFyKSaYzpZ3ccwUTvyen0npxO78npwumehGNNoaam2B1AENJ7cjq9J6fTe3K6sLknEVtTUEopdbpIrikopZSqRJOCUkqpcpoUlFJKldOk4CUiCSLyfyLyLxG5xe54goGItBeR10Rktt2xBAsRudL7N/IfEbnY7niCgYh0FZFXRGS2iNxtdzzBwvuZkikiY+2OxRdhnRREZJqI7BeRtZXKR4vIBhHZLCIPe4uvBmYbYyYAl9d7sPXEl3tijNlqjBlvT6T1x8d78r73b+Qu4AY74q0PPt6TLGPMXcD1wGA74q0PPn6eAPwaeLt+o6y7sE4KwHRgdMUCEXECLwKXAt2Am0SkG9YOcGV7R5fWY4z1bTo1vyeRYjq+35NHva+Hq+n4cE9E5HLgY+qwJH4ImE4N74mIXASsB/bXd5B1FdZJwRizEMitVDwA2Oz9FlwMvAVcAeRgJQYI4/vi4z2JCL7cE7E8Ccw1xqyo71jri69/J8aYD40xlwJh2/Tq4z0ZAZwH3AxMEJGQ+UwJ6CY7QSqVkzUCsJLBQOB5YLKIXEbgp68HmyrviYg0Af4K9BaRR4wxj9sSnT3O9HdyH3AhkCgiGcaYV+wIziZn+jsZgdX8GkN41xSqUuU9McZMAhCR24GDxhiPDbHVSiQmhSoZY04Ad9gdRzAxxhzCajtXXsaY57G+QCgvY8wCYIHNYQQlY8x0u2PwVchUafxoF9C6wvM0b1kk03tyOr0np9N7crqwuyeRmBSWAR1FpJ2IRAM3Ah/aHJPd9J6cTu/J6fSenC7s7klYJwURmQUsBjqLSI6IjDfGuIFJwHwgC3jbGLPOzjjrk96T0+k9OZ3ek9NFyj3RBfGUUkqVC+uaglJKKd9oUlBKKVVOk4JSSqlymhSUUkqV06SglFKqnCYFpZRS5TQpKFUFEUkSkXvOcswIEZlTXzEpVR80KShVtSSg2qSgVDjSpKBU1Z4AOojI9yLytPexVkTWiMhpm+uISH8RWSkiHbw7bk0Tke+8ZVd4j7ldRP4rIvNEZJOIPOUtd4rI9ArXf6Cef1elyukqqUpV7WGguzGml4hcg7VabE+gKbBMRBaWHSgig4AXgCuMMTtE5G/AF8aYO0UkCfhORD7zHt4L6A0UARtE5AUgBUg1xnT3Xi+pnn5HpU6jNQWlzm4IMMsYU2qM2Qd8BfT3vtYVmAKMM8bs8JZdDDwsIt9jLSkdC6R7X/vcGJNnjCnE2pmrDbAVaC8iL4jIaOBoffxSSlVFk4JSdbMHKMT69l9GgGuMMb28j3RjTJb3taIKx5UCLmPMYaxayAKsGsnUwIetVNU0KShVtWNAQ+/PXwM3eNv+mwHDgO+8rx0BLgMe9+5ABtaKmfeJiACISMWEcRoRaQo4jDHvYu393Mefv4hSvtA+BaWqYIw5JCLfiMhaYC6wGlgFGOAhY8xeEeniPXafiIwF5orIncCfgX8Aq717824DxlbzdqnAvyvs4/tIYH4rpc5Ol85WSilVTpuPlFJKldOkoJRSqpwmBaWUUuU0KSillCqnSUEppVQ5TQpKKaXKaVJQSilVTpOCUkqpcv8Pze4Rh4rQoN8AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "trigram_freqs = [freq for token, freq in trigram_vocab.token_freqs]\n",
    "axes = plt.gca()\n",
    "axes.set_xlabel('tokens')\n",
    "axes.set_ylabel('freqs')\n",
    "axes.set_xscale('log')\n",
    "axes.set_yscale('log')\n",
    "axes.plot(freqs)\n",
    "axes.plot(digram_freqs)\n",
    "axes.plot(trigram_freqs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(('the', 'time', 'traveller'), 53), (('the', 'time', 'machine'), 24), (('the', 'medical', 'man'), 22), (('it', 'seemed', 'to'), 14), (('it', 'was', 'a'), 14)]\n"
     ]
    }
   ],
   "source": [
    "print(trigram_vocab.token_freqs[:5])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Random Sampling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "def seq_rand_data_iter(corpus, batch_size, num_steps):\n",
    "    corpus = corpus[random.randint(0, num_steps):]\n",
    "    num_examples = ((len(corpus) - 1) // num_steps)\n",
    "    example_indices = list(range(0, num_examples*num_steps, num_steps))\n",
    "    random.shuffle(example_indices)\n",
    "    \n",
    "    data = lambda pos: corpus[pos: pos+num_steps]\n",
    "    \n",
    "    num_data = num_examples // batch_size\n",
    "    for i in range(0, num_data*batch_size, batch_size):\n",
    "        indices = example_indices[i: (i+batch_size)]\n",
    "        X = [data(j) for j in indices]\n",
    "        Y = [data(j+1) for j in indices]\n",
    "        yield nd.array(X), nd.array(Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[[78. 79. 80. 81. 82.]\n",
      " [23. 24. 25. 26. 27.]]\n",
      "<NDArray 2x5 @cpu(0)>\n",
      "\n",
      "[[79. 80. 81. 82. 83.]\n",
      " [24. 25. 26. 27. 28.]]\n",
      "<NDArray 2x5 @cpu(0)>\n"
     ]
    }
   ],
   "source": [
    "seq = list(range(0, 100))\n",
    "for X, y in seq_rand_data_iter(seq, 2, 5):\n",
    "    print(X)\n",
    "    print(y)\n",
    "    break"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Sequential Sampling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "def seq_cons_data_iter(corpus, batch_size, num_steps):\n",
    "    offset = random.randint(0, num_steps)\n",
    "    \n",
    "    num_indices = ((len(corpus)-offset-1)//batch_size)*batch_size\n",
    "    Xs = nd.array(corpus[offset:offset+num_indices])\n",
    "    Ys = nd.array(corpus[offset+1:offset+1+num_indices])\n",
    "    Xs = Xs.reshape((batch_size, -1))\n",
    "    Ys = Ys.reshape((batch_size, -1))\n",
    "    num_batches = Xs.shape[1] // num_steps\n",
    "    for i in range(0, num_batches*num_steps, num_steps):\n",
    "        X = Xs[:, i:(i+num_steps)]\n",
    "        y = Ys[:, i:(i+num_steps)]\n",
    "        yield X, y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[[ 6.  7.  8.  9. 10. 11.]\n",
      " [52. 53. 54. 55. 56. 57.]]\n",
      "<NDArray 2x6 @cpu(0)>\n",
      "\n",
      "[[ 7.  8.  9. 10. 11. 12.]\n",
      " [53. 54. 55. 56. 57. 58.]]\n",
      "<NDArray 2x6 @cpu(0)>\n"
     ]
    }
   ],
   "source": [
    "for X, y in seq_cons_data_iter(seq, 2, 6):\n",
    "    print(X)\n",
    "    print(y)\n",
    "    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_time_machine_corpus(max_tokens=-1):\n",
    "    lines = read_time_machine()\n",
    "    tokens = tokenize(lines)\n",
    "    vocab = Vocab(tokens)\n",
    "    corpus = [vocab[tk] for line in tokens for tk in line]\n",
    "    if max_tokens > 0: corpus = corpus[:max_tokens]\n",
    "    return corpus, vocab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "corpus, vocab = load_time_machine_corpus()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "def SeqDataLoader(object):\n",
    "    def __init__(self, batch_size, num_steps, use_random, max_tokens):\n",
    "        if use_random:\n",
    "            data_iter = seq_rand_data_iter\n",
    "        else:\n",
    "            data_iter = seq_cons_data_iter\n",
    "        self.corpus, self.vocab = load_time_machine_corpus(max_tokens)\n",
    "        self.get_iter = lambda : data_iter(self.corpus, batch_size, num_steps)\n",
    "    def __iter__(self):\n",
    "        return self.get_iter()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}