LanguageModels.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "from mxnet import nd, autograd, init, context, gluon\n",
    "from mxnet.gluon import nn\n",
    "from IPython import display\n",
    "import re\n",
    "import collections"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_time_machine():\n",
    "    with open('./data/timemachine.txt', 'r') as f:\n",
    "        lines = f.readlines()\n",
    "    return [re.sub('[^A-Za-z]+', ' ', line.strip().lower()) for line in lines]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['the time machine by h g wells ', '', '', '', '']\n"
     ]
    }
   ],
   "source": [
    "lines = read_time_machine()\n",
    "print(lines[:5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tokenize(lines):\n",
    "    tokens = [line.split(' ') for line in lines]\n",
    "    return tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['the', 'time', 'machine', 'by', 'h', 'g', 'wells', ''], ['']]\n"
     ]
    }
   ],
   "source": [
    "tokens = tokenize(lines)\n",
    "print(tokens[:2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def count_freqs(lines):\n",
    "    flat = [tk for line in lines for tk in line ]\n",
    "    return collections.Counter(flat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('the', 2261), ('time', 200), ('machine', 85), ('by', 103)]\n"
     ]
    }
   ],
   "source": [
    "token_freqs = count_freqs(tokens)\n",
    "print(list(token_freqs.items())[:4])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Vocab(object):\n",
    "    def __init__(self, tokens, min_freq=0):\n",
    "        self.token_freqs = count_freqs(tokens)\n",
    "        self.token_freqs = sorted(self.token_freqs.items(), key=lambda x:x[0])\n",
    "        self.token_freqs = sorted(self.token_freqs, key=lambda x:x[1], reverse=True)\n",
    "        \n",
    "        self.unq, self.unq_tokens = 0, ['<unk>']\n",
    "        self.unq_tokens += [token for token, freq in self.token_freqs if freq > min_freq]\n",
    "        self.idx_to_tokens, self.tokens_to_idx = [], dict()\n",
    "        \n",
    "        for token in self.unq_tokens:\n",
    "            self.idx_to_tokens.append(token)\n",
    "            self.tokens_to_idx[token] = len(self.idx_to_tokens) - 1\n",
    "        \n",
    "    @staticmethod\n",
    "    def count_freqs(lines):\n",
    "        flat = [tk for line in lines for tk in line ]\n",
    "        return collections.Counter(flat)\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.idx_to_tokens)\n",
    "    \n",
    "    def __getitem__(self, tokens):\n",
    "        if not isinstance(tokens, (list, tuple)):\n",
    "            return self.tokens_to_idx[tokens]\n",
    "        return [self.tokens_to_idx[token] for token in tokens]\n",
    "    \n",
    "    def to_tokens(self, indices):\n",
    "        if not isinstance(indices, (list, tuple)):\n",
    "            return self.idx_to_tokens[indices]\n",
    "        return [self.idx_to_tokens[index] for index in indices]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "lines = read_time_machine()\n",
    "tokens = tokenize(lines)\n",
    "vocab = Vocab(tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('the', 2261), ('', 1282), ('i', 1267), ('and', 1245), ('of', 1155)]\n"
     ]
    }
   ],
   "source": [
    "print(vocab.token_freqs[:5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deXxU5d3+8c93JishCYQEgbAnrFIBjVA3SqsiVhG3KrWtVVGqVWsfW1tr26f+qt0361algrj00SpWBRWpWMWlVgiIyCKLrGGXQFhDtvv3R4ZphCRkmcmZM3O9X6+8yNyzcHEMXN5nuY855xAREQEIeB1ARERih0pBRETCVAoiIhKmUhARkTCVgoiIhKkUREQkLMnrAK2Rm5vrevfu7XUMERFfWbBgwafOubz6nvN1KfTu3Zvi4mKvY4iI+IqZrW/oOV/uPjKzcWY2uayszOsoIiJxxZel4Jyb6ZyblJ2d7XUUEZG44stSEBGR6FApiIhImEpBRETCVAoiIhKmUhARkTCVgoiIhKkUREQkTKUgIiJhKgUREQnzZSlomQsRkejwZSlomQsRkejwZSmIiEh0qBRERCRMpSAiImEqBRERCVMpiIhImEpBRETCVAoiIhKmUhARkbAkrwO0xurt+xh//zste7MZQYNgwAiYEQzYZ76v/RWSAgECgdrX1v4aer7u96HXHh5LCj2fFDBSk4KkJgdITQqQkhSofXz419B4alIw9FzoKzlIRkoQM4vsBhMROQZfl0JSwOiYkdKi99Y4qKlxVNc4qp2joqqGaudqx5yjuqb2+aqaGmocta+rcdS4z/5a+z3hz6l9j2v1ny0zNYlBXbMY1DWTwd2yGNQ1i/7HZZKWHGz1Z4uINMSca/0/YF4pKipyxcXFXseo1+FyqKiu4VBlNYeqakJf1VQc/r6y9vGhqprQWO335ZXVbCw9yLIte/h4yx72V1QDEDAoyGsfKossBnfL4oT87BYXo4gkJjNb4Jwrqu85X88UYlkgYKQEjJSkAO1TW76Za2ocG0oPsHzLHpZv2cOyLXtYsH4XMz7cDEB6cpAnJo6gqHdOpKKLSAKLmZmCmQ0CbgFygdedc3851ntieaYQbbsPVLBsyx5+/PwSdu47xPQbTqX/cZlexxIRH2hsphDVs4/MbKqZbTezJUeMjzWzFWa22sxuB3DOLXfOXQ9cBpwWzVzxoEO7FE4tyOXxa0aQlhzkyinz2LT7oNexRMTnon1K6jRgbN0BMwsCDwDnAoOBr5rZ4NBzFwAvA69EOVfc6JHTjseuGcH+Q1VcOeV9du2v8DqSiPhYVEvBOfcWUHrE8AhgtXNujXOuAngaGB96/Qzn3LnA1xr6TDObZGbFZla8Y8eOaEX3lUFds/jrN4vYuOsgV0+bz4GKKq8jiYhPeXHxWj6wsc7jEiDfzEab2b1m9jCNzBScc5Odc0XOuaK8vLxoZ/WNz/ftxL0ThrG4ZDc3/m0hldU1XkcSER+KmSuanXNvOue+45z7lnPuAa/z+NHYIV2568IhvLFiBz98bjGxchKBiPiHF6ekbgJ61HncPTTWZGY2DhhXWFgYyVxx4Wsje/Hp3gr+NGcleZmp/OjcQV5HEhEf8WKmMB/oZ2Z9zCwFmADMaM4H6B7NjfvOmYV8/fM9eXjuGh55e43XcUTER6J9SupTwHvAADMrMbOJzrkq4CZgNrAceMY5tzSaORKNmfH/LhjCuUO6cPfLy/nOUx+wfW+517FExAdi5uK15qiz++i6VatWeR0nZlVU1fDgm6t58I1PSEsOcPu5g5hwcg8CAS20J5LIGrt4zZelcFgiX9HcHJ/s2MePn/+I/6wp5aReHfnlRZ9jQBdd/SySqDy7olliQ0Fee5667vP84StDWbNjH+fd+za/efVjDoYW2hMROcyXpWBm48xscllZmddRfMPMuOSk7rz+vdFcNDyfv7z5CZdPfo+95ZVeRxORGOLLUtDZRy2Xk5HC774ylMnfOIllm/cw8bFizRhEJMyXpSCtN+b4Lvzp8mHMX1fK9U8uoKJKV0CLiEohoY0b2o1fXfQ55q7cwXf//gFVWhpDJOH58iY7uqI5ciaM6Mm+Q1Xc/fJy2qV8xG8vOUGnrIokMF/OFHRMIbKuPaMv3z2rH9MXlHDnzKWU7Dqg3UkiCcqXMwWJvFvO7Mfe8iqmvLOWx99bjxl0ykilS3Yqg7pkcecFx5PRituKiog/6G+5ALWnrP7kvEGcNeg41u/cz9Y95WzbU87m3eU8t7CEsoOVPPT1k7RrSSTOqRQkzMw4paATpxR0+sz4lHfWctdLy/jjayv5/jkDPEonIm3Bl8cUdPFa27rmtN5MOLkH97+xmhcXNWuVcxHxGV+Wgg40ty0z4+fjhzCiTw63TV/MBxt2eR1JRKJEC+JJk5Xur2D8A+9wsKKGC4d1IyM1iYzUIO1SkmifmkS7lCDt05Io6pVDSpIv/39DJCE0tiCejilIk+VkpDDlmydzw5ML+L95GzjQwPIYZw8+jr9eWe/Pm4jEOJWCNEv/4zJ5/XujAaiucRysrObAoSr2V1Sz/1AVMxdv5uG5a5i7cgdf6J/nbVgRaTbN8aXFggGjfWoSnbPS6JObwZD8bG49uz+9O7Xj5zOXUqllM0R8x5eloLOPYldqUpCfnj+YT3bs57F/r/M6jog0ky9LQWcfxbYvDezMF/rn8ec5q9ix95DXcUSkGXxZChLbzIz/HTeYg5XV/PG1FV7HEZFmUClIVBTktedrI3vybHEJG0sPeB1HRJpIpSBRc/3oAgJmPDT3E6+jiEgTqRQkarpmp3NpUXeeLS5ha1m513FEpAlUChJVN3yhgBrnNFsQ8QlfloJOSfWPHjntuPjEfJ6at4HtezVbEIl1viwFnZLqL98eXUhldQ0TpxUzfUEJByqqvI4kIg3wZSmIv/TOzeDXl5zA3vJKvv/sh4z8xevMXbnD61giUg+VgrSJy4p68Mb3R/PMt06he047vvVEMfPWlnodS0SOoFKQNmNmjOiTwxMTR9CtQzoTp81nySYdFxKJJSoFaXO57VN5cuJIstKTuXLqPFZv3+t1JBEJUSmIJ7p1SOfJa0cSMONrj7yvq55FYoRKQTzTJzeDJ68dQXllDZc9/B6/n72CeWtL8fPdAEX8TqUgnhrYJYvHr6k9xvCXuZ9w2cPvce1jxXy6T6urinhB92iWmLGnvJJn5m/kt7NXkJWWxCUnducL/fM4tTDX62gicaWxezRrpiAxIystmWvP6MvMm05nYJcsHn13HVc88j6vLtnqdTSRhOHLUtAyF/FtQJdMnrx2JB/+bAy57VN4+aMtXkcSSRi+LAUtc5EY0lOCnDnwON5csZ2KKt3vWaQt+LIUJHGcNfg49pZX6epnkTaiUpCYdnphLmnJAV5bpuMKIm1BpSAxLT0lyOmFecxZvl3XL4i0AZWCxLyxQ7qwafdBrp42n9Xb93kdRySuqRQk5l08PJ87vjyQBet3cdGD77Jtj27WIxItKgWJeYGAMWlUATNuOp2KqhrunLHU60gicUulIL7RJzeDW87qx6wlW7nt2Q/57asfs2D9Lh1rEImgJK8DiDTHdWf05T9rSpm1ZCvlldU8+OYnDO2ezW8vHcqALplexxPxPa19JL6171AVMz/czO9nr2D3wUpO6tmRy0/uwUXD8wkEzOt4IjGrsbWPVAriezv3HWLqu2t5bdk2Vm7bx8AumQzJzyYtuXbvaH6HdlwwrBv5HdI9TioSG1QKkhBqahzTF5YwfUEJ63fup6ra4YDS/RWYwWkFufzyos/Rs1M7r6OKeEqlIAltY+kBnv9gE4+8vYbqGkfXDukU5GWQlZbMgC6ZXDGyJ+1SdHhNEkdjpRBTfxPM7ELgPCALmOKc+6fHkSQO9Mhpx3fO7Mf4Yd148I1P2H2wglXb97GvvIpnF5SwZFMZ90wY7nVMkZgQ9VIws6nA+cB259yQOuNjgT8DQeAR59yvnXMvAC+YWUfg94BKQSKmV6cMfnPpCZ8Zu2fOSu6Zs4r1pQf41qi+jB3S1aN0IrGhLa5TmAaMrTtgZkHgAeBcYDDwVTMbXOclPwk9LxJVN36xkNvOGcCeg5Vc/+RCrcYqCS/qpeCcews48m/aCGC1c26Nc64CeBoYb7V+A8xyzi2s7/PMbJKZFZtZ8Y4dO6IbXuJecjDAjV8s5KWbzyAzNYmn52/wOpKIp7w6ppAPbKzzuAQYCdwMnAVkm1mhc+6hI9/onJsMTIbaA81tkFUSQHpKkPOHduO5hSVkpCQxtEcHBnbJ5PhuWZjpmgdJHDF1oNk5dy9wr9c5JDHdcmY/du2v4NkFG3niP+sBGDP4OG4d05+BXbI8TifSNrwqhU1AjzqPu4fGmsTMxgHjCgsLI51LEliX7DQe+sZJHKyoZkvZQWYt2cqfX1/FP5dtY2iPDlxyYj45GSkMOC6TvnntCeqqaYlDbXKdgpn1Bl46fPaRmSUBK4EzqS2D+cAVzrlmLX+p6xQk2nbtr+D5Dzbx9PwNrNz233s55LZP4a9XFjG8Z0cP04m0jKcXr5nZU8BoIBfYBvzMOTfFzL4M3EPtKalTnXO/aO5nqxSkrTjn2FB6gP2HqvnPmp1MeWctldU1zP7uKDpmpHgdT6RZ4u6K5jq7j65btWqV13EkAS3ZVMbFD/6bQd2yeHLiCDLTkr2OJNJkcVcKh2mmIF7659Kt3PC3heRkpNC9YzoXD8/nipG9dKxBYl5jpaCb7Ii00Jjju/DExBGc2LMDldU1/PTFpVz84Ls8U7yRJZvKvI4n0iK+nClo95HEGuccMz7czF0vLePTfbWrsk696mS+OKCz19FEjtKqmYKZ/dbMssws2cxeN7MdZvb1yMdsOufcTOfcpOzsbC9jiISZGeOH5fPOD7/EnFtH0S07nfv/tdrrWCLN1pTdR2Occ3uoXdRuHVAI3BbNUCJ+lZYcpLBzJt84pRcL1u9iY+kBryOJNEtTSuHwBW7nAc8657SzVOQYvjykKwGDW59ZROn+Cq/jiDRZU0rhJTP7GDgJeN3M8oDy6MZqnJmNM7PJZWXqJ4lNPTu1488ThrO4pIyiu1/jHwtLKK+s9jqWyDE16UCzmeUAZc65ajPLADKdc1ujnu4YdEqqxLolm8q46tF5fLqvgn6d2zPrljNICuqkP/FWq65TMLOLG3veOfePVmRrFZWC+EHp/goefXct9/1rNSf37sg9E4aT3yHd61iSwFp7O86JwKnAv0KPvwj8G9gBOMCzUhDxg5yMFG49uz/JwQD3zFnJVVPnMf36U8lup6ugJfY0pRSSgcHOuS0AZtYVmOacuzqqyUTiiJnxnTP7UdS7I9+cOo9JTxRz69n9SQoGGNajg66ClpjRlN1Hy51zg+o8DgBL6461NV28Jn724qJN3PL0ovDjAcdlcs+EYQzqqns2SNto7TGF+4F+wFOhocupvZXmzRFN2QI6piB+tWrbXrbvPcT7a3Zyb+git3FDu3H1ab05UctxS5S1ekE8M7sIGBV6+JZz7vkI5msxlYLEg2Wb93DH8x+xaONuAKZdfTKjtTyGRFEkSqEX0M85N8fM2gFB59zeCOdsNpWCxJPNuw8y/oF32bH3EF//fE9+fsEQAjrWIFHQ2rWPrgOmAw+HhvKBFyIXT0QAunVI56nrRjIkP4sn/7OBP762ksrqGq9jSYJpylU0NwKnAXsAnHOrAM1tRaKgsHMmM286nTP65XL/G6sZ8Ys57Nx3yOtYkkCaUgqHnHPhxVtC91f2dL1tLXMh8czMuP+KE/nRuQPZfbCSr0+Zx97ySq9jSYJoSinMNbM7gHQzOxt4FpgZ3ViN09LZEu+y05P51hcKuPTE7izfsofP3flPvjHlfWZ9tIWKKu1SkuhpyimpAWqvah4DGDAbeMTFwN15dKBZ4l1NjeOZ4o08NPcT1u2sXYa7d6d2fP+cAZx/QjeP04lftfjsIzMLAo87574WrXCtoVKQRFFRVcPGXQe466VlvLliBwDdO6bzs3HHc/bg4zxOJ37T4rOPnHPVQC8zS4lKMhFpkpSkAAV57Zl29Qjm3DqKy4t6ULLrINc9Xsxf3vyEAxVVXkeUONGU3UePA4OAGcD+w+POuT9GN9qxaaYgiezNFdu56tH5ABzfLYsXbjyNZC3LLU3QopmCmT0R+vYC4KXQazPrfImIh0YP6My8O86kIC+DpZv38N066ymJtFRjq6SeZGbdgA3AfW2Up0nqLIjndRQRT3XOSmPWLaMY+cs5vPzRFq5eV0pR7xyvY4mPNTbXfAh4HegPFNf5WhD61TM6JVXkv1KSAsy46XQALn3oPe54/iOPE4mfNVgKzrl7Q8tjP+qc61vnq49zrm8bZhSRY+iR044/XT6Uod2z+b/3N3Dxg+9SpSUypAWOeVTKOXdDWwQRkda5aHh3Hvz6SRTkZbBww276/WQWMz/c7HUs8RmdqiASR/I7pPP8jadxx5cH0j41iZuf+oClm7UcjDSdSkEkzmSlJTNpVAF3XzgEgPPufYclm1QM0jQqBZE4NX5YPj8cOxCAK6fOY86ybR4nEj9QKYjEsRtGF/DDsQOpqKrhpy8u4Yq//oc3Vmz3OpbEsMauUxCROHDD6AIA3vh4O4s27uaXLy9n0Ybd3PSlQl0BLUfRT4RIArhhdAHPXH8KV4zoScmug/z59VU88vZa3lixnfLKaq/jSQxp0j2aY02dK5qvW7VqlddxRHxl255yTvnV69SE/urfdeEQvvH5Xt6GkjbV4qWzY50WxBNpmY2lByg7WMnlD7/H/opqstKSmP0/o+iane51NGkDLV46W0TiU4+cdgzJz+YPlw1jwsk92FNexU9fWOp1LIkBKgWRBDZ2SBd+fN4gAOYs38aG0N3dJHGpFEQSXGZaMn/4ylAARv3uDeau3OFxIvGSTkkVEc47oSsA33v2Qx7412pmL93K5/Kz+eqInh4nk7ammYKIkJYc5KLh+YzoncOaT/fz/MJN3PXSMq9jiQdUCiICQCBgPHP9KRT/5CxuPrOQAxXVnHDnbJ5bUOJ1NGlDKgUROcqFw/KZNKovVTWOf328nTU79uHn09el6VQKInKUbh3SuePLg+iTm8HLH23hS3+YqzWTEoQONItIg+6/4kTmryvlB9MXM3fFDgJmpCQFGNE7hyStmxSX9F9VRBrUJzeDC4flk54c5LH31nPVo/O54q/v8+rSrV5HkyjRTEFEGpWSFOC1W0exfe8h9pZX8c2p8/hgw256d8rg+G5ZmJnXESWCNFMQkWPq3rEdJ/bsyKkFnUhNCjDlnbWcf987zFtb6nU0iTCVgog0WXIwwMvfOZ17Lh8GwPqdB3DO6cykOBIzpWBmfc1siplN9zqLiDSssHMmY4d0wQx+8Nxi+vzoFfr86BXue13L2MeDqJaCmU01s+1mtuSI8bFmtsLMVpvZ7QDOuTXOuYnRzCMikZGWHOSey4fx3bP68d2z+nFcViqLNu72OpZEQLQPNE8D7gcePzxgZkHgAeBsoASYb2YznHO6pl7ER8YPyw9/X7xuF0s2l3HnjNrltzNSg9z8pX6kJQe9iictFNVScM69ZWa9jxgeAax2zq0BMLOngfFAk0rBzCYBkwB69tRiXSKx4NTCTiwu2c0/FpZQXePYX1HNKX1zOb1frtfRpJm8OKaQD2ys87gEyDezTmb2EDDczH7U0Judc5Odc0XOuaK8vLxoZxWRJvj26EIW33kOi+88h+dvPA2A3QcrPE4lLREz1yk453YC13udQ0RaJzs9GYD/fXEpv3n1YwBu/lI/Livq4WUsaSIvZgqbgLo/Hd1DY01mZuPMbHJZWVlEg4lI63XOTOXGLxYwun8eJ/fKYff+St5Z9anXsaSJvJgpzAf6mVkfastgAnBFcz7AOTcTmFlUVHRdFPKJSCuYGbedMzD8+Pz73mbfoSoPE0lzRLUUzOwpYDSQa2YlwM+cc1PM7CZgNhAEpjrndMdwkTjVPjWJd1d/yujfvREeCwaMu8YP4dRCHYiONdE+++irDYy/ArzS0s81s3HAuMLCwpZ+hIi0kWtO68PLWVs+M/bios3MW1eqUohBMXOguTm0+0jEP8Yc34Uxx3f5zNisJVs5WFntUSJpjC9LQUT8rV1KkA/W72bKO2s/M94tO41zP9fVo1QCPi0F7T4S8be+uRnMW1fKvHVHr7K6+M4xZKUle5BKwKeloN1HIv72zLdOYX/FZ3cfPVu8kbtfXk55RbVKwUO+LAUR8bekYIDs9M9eJtWhXQoAh6pqvIgkISoFEYkJqUm1JfHI22vIyUgNjycFjcuKepCXmdrQWyWCfFkKOqYgEn/65GaQmhTgsffWH/VcalKAa8/o60GqxOPLUtAxBZH4MyQ/m+U/H/uZsaoaR/+fzOJAhU5fbSu+LAURiU+BgH3mcUrACAaMQ1UqhbYSM7fjFBGpT1pSgPJKHXxuK5opiEhMS0sO8tzCknpXWk0KGndfOIThPTt6kCw++bIUdKBZJHHcMLqA4nW7jhqvqqlhzvLtLFi/S6UQQb4sBR1oFkkc157Rl2vPOHq8vLKagT99lYpq7VqKJB1TEBFfSg7W/vNVWeU8ThJfVAoi4kvB0JlJlZopRJQvdx+JiAAkB41dByrYvPtgg6/p2C6F9JRgG6byN5WCiPhWRkoSf3t/A397f0ODr+mTm8Eb3x/ddqF8zpeloLOPRARg8pUnsXr7vgaff/mjrcxbu7MNE/mfL0tBZx+JCMBJvXI4qVdOg8+v23mAf68++voGaZgONItI3EoOGFU1Dud0hlJTqRREJG4lhU5bra5RKTSVSkFE4lZSsHaBvSqVQpOpFEQkbiUHav+JUyk0nS8PNIuINMXhmcJVU+cRPGJZ7vrkd0zn95cOPWoJ70Tiy5mCmY0zs8llZWVeRxGRGHZKQSdOL8xtUiFs3VPOPxZuYm95VRski13m56PyRUVFrri42OsYIhIHpr27ljtnLmPhT88mJyPF6zhRZWYLnHNF9T3ny5mCiEikBXWmEqBSEBEBIGi1u5hqfLz3JBJUCiIiQGiioJmC1wFERGJBIDRTUCmIiEj4DCXtPhIRkXApaKYgIiLh3UeJPlPQFc0iIvx3pjDro610yd7d7PebGaP659I5My3S0dqUL0tBN9kRkUjrnJkKwB9eW9niz7jq1N7cecHxkYrkCV+Wgm6yIyKRVtQ7h/k/PovyyuoWvf+iB99t8XtjiS9LQUQkGvJCs4WWSAoEiIfDETrQLCISAWbxcZBapSAiEgEBM/xfCSoFEZGI0ExBRETCzNAxBRERqRUww8/3pzlMpSAiEgEGxMMKGSoFEZEICJjpmIKIiNQyQ2cfiYhILdMxBREROSxgUFPjdYrWUymIiERA7cVrmimIiEhIPJx9FDML4plZBvAgUAG86Zz7m8eRRESaTNcpNIGZTTWz7Wa25IjxsWa2wsxWm9ntoeGLgenOueuAC6KZS0Qk0gKB+LiiOdozhWnA/cDjhwfMLAg8AJwNlADzzWwG0B34KPQy/y9KLiIJxTCWbC7j1r8vapPf74qRPSnqnRPxz41qKTjn3jKz3kcMjwBWO+fWAJjZ08B4aguiO7CIRmYwZjYJmATQs2fPyIcWEWmBUf1zmfHhZuavL22T32/M8V2i8rleHFPIBzbWeVwCjATuBe43s/OAmQ292Tk3GZgMUFRUFAeTNRGJB7edM5DbzhnodYxWi5kDzc65/cDVXucQEUlkXpySugnoUedx99BYk5nZODObXFZWFtFgIiKJzotSmA/0M7M+ZpYCTABmNOcDnHMznXOTsrOzoxJQRCRRRfuU1KeA94ABZlZiZhOdc1XATcBsYDnwjHNuaTRziIhI00T77KOvNjD+CvBKSz/XzMYB4woLC1v6ESIiUg9fLnOh3UciItHhy1IQEZHo8GUp6OwjEZHoMD8v4GRmO4D1QDZwZEMcOXbk41zg06gGrD9XpN97rNc19nxDzx1r2x05lijbsrHXRGpbQmxvz+a8r6U/m80Z9/Pfcy+3ZS/nXF69n+Sc8/0XMPlYY/U8LvYiV6Tfe6zXNfZ8Q88da9sdOZYo27I526yl2zLWt2dz3tfSn83mjPv577mX27KxL1/uPqpHfctiHDnW4NIZUdSa37Op7z3W6xp7vqHnmrLt2np7xsK2bOw1ftqWrfk9m/O+lv5sNmfcz3/PvdyWDfL17qPWMLNi51yR1znigbZlZGl7Ro62ZfPFy0yhJSZ7HSCOaFtGlrZn5GhbNlPCzhRERORoiTxTEBGRI6gUREQkTKUgIiJhKoUQM8sws8fM7K9m9jWv8/iZmfU1sylmNt3rLH5nZheGfib/bmZjvM7jZ2Y2yMweMrPpZnaD13liVVyXgplNNbPtZrbkiPGxZrbCzFab2e2h4YuB6c6564AL2jxsjGvOtnTOrXHOTfQmaexr5rZ8IfQzeT1wuRd5Y1kzt+Vy59z1wGXAaV7k9YO4LgVgGjC27oCZBYEHgHOBwcBXzWwwtXeAO3zv6Oo2zOgX02j6tpTGTaP52/Inoefls6bRjG1pZhcAL9OKpfvjXVyXgnPuLaD0iOERwOrQ/81WAE8D44ESaosB4ny7tEQzt6U0ojnb0mr9BpjlnFvY1lljXXN/Lp1zM5xz5wLaRdyARPzHL5//zgigtgzygX8Al5jZX/DmUnk/qndbmlknM3sIGG5mP/Immu809HN5M3AWcKmZXe9FMB9q6OdytJnda2YPo5lCg6J65zU/cc7tB672Okc8cM7tpHYfuLSSc+5e4F6vc8QD59ybwJsex4h5iThT2AT0qPO4e2hMmk/bMnK0LSNH27IVErEU5gP9zKyPmaUAE4AZHmfyK23LyNG2jBxty1aI61Iws6eA94ABZlZiZhOdc1XATcBsYDnwjHNuqZc5/UDbMnK0LSNH2zLytCCeiIiExfVMQUREmkelICIiYSoFEREJUymIiEiYSkFERMJUCiIiEqZSEKmHmXUws28f4zWjzeyltsok0hZUCiL16wA0Wgoi8UilIFK/XwMFZrbIzH4X+lpiZh+Z2VE3uzGzk83sAzMrCN3Fb6qZzQuNjQ+95ioz+4eZvWpmq8zstzrqVn4AAAGNSURBVKHxoJlNq/P5/9PGf1aRMK2SKlK/24EhzrlhZnYJtau+DgVygflm9tbhF5rZqcB9wHjn3AYz+yXwL+fcNWbWAZhnZnNCLx8GDAcOASvM7D6gM5DvnBsS+rwObfRnFDmKZgoix3Y68JRzrto5tw2YC5wcem4QMBkY55zbEBobA9xuZouoXao5DegZeu5151yZc64cWAb0AtYAfc3sPjMbC+xpiz+USH1UCiKtswUop/b//g8z4BLn3LDQV0/n3PLQc4fqvK4aSHLO7aJ2FvImtTOSR6IfW6R+KgWR+u0FMkPfvw1cHtr3nweMAuaFntsNnAf8ysxGh8ZmAzebmQGYWd3COIqZ5QIB59xz1N6L+cRI/kFEmkPHFETq4ZzbaWbvmtkSYBawGPgQcMAPnHNbzWxg6LXbzOx8YJaZXQPcBdwDLDazALAWOL+R3y4feDT0WgDdwlQ8o6WzRUQkTLuPREQkTKUgIiJhKgUREQlTKYiISJhKQUREwlQKIiISplIQEZEwlYKIiIT9f5Pi7G6vWCNOAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "freqs = [freq for token, freq in vocab.token_freqs]\n",
    "axes = plt.gca()\n",
    "axes.plot(range(len(freqs)), freqs)\n",
    "axes.set_xlabel('tokens')\n",
    "axes.set_ylabel('freqs')\n",
    "axes.set_xscale('log')\n",
    "axes.set_yscale('log')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "bigram_tokens = [[pair for pair in zip(line[:-1], line[1:])] for line in tokens]\n",
    "bigram_vocab = Vocab(bigram_tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(('of', 'the'), 297), (('in', 'the'), 161), (('i', 'had'), 126), (('and', 'the'), 104), (('i', 'was'), 104)]\n"
     ]
    }
   ],
   "source": [
    "print(bigram_vocab.token_freqs[:5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAeTklEQVR4nO3deXxV1b338c/vJCFhCAnzHAETGRXQgIheh6o4otbaCtpeay2oVTvc20Frn9rX7dzex9aptdSB2lqHWi1gtT6KVatQGRRlVAEhQBAQISFAIMPv+eOcHCMk4RySk32G7/v1ykvO2vuc/FgGvqy99l7L3B0RERGAUNAFiIhI8lAoiIhIlEJBRESiFAoiIhKlUBARkSiFgoiIRGUHXUBr9OzZ0wcPHhx0GSIiKWXJkiUfunuvpo6ldCgMHjyYxYsXB12GiEhKMbMNzR3T5SMREYlSKIiISJRCQUREohQKIiISpVAQEZEohYKIiEQpFEREJColQ8HMppjZzIqKiqBLERFJKykZCu4+191nFBQUBF2KiEhaSclQEBGRxFAoiIhIlEJBRESiFAoiIhKlUBARkSiFgoiIRCkUREQkSqEgIiJRCgUREYlSKIiISJRCQUREohQKIiISlZKhoFVSRUQSIyVDQaukiogkRkqGgoiIJIZCQUREohQKIiISpVAQEZGo7KALaI16d6r21x7Re3OzQ+RkKRNFRBpL6VBYUV7J6NueO6L3dszJ4qIx/Zl2YhFjBhZgZm1cnYhI6knpUOhXkMet5484ove+t203c94q57HFGxneN58rTizi4rEDKOiY08ZVioikDnP3oGs4YqWlpb548eIjfv/u6hrmvFXOn18vY0V5JXk5IS48rj/TJhRxfFGhRg8ikpbMbIm7lzZ5LJNDobFlmyr488Iy5izdzJ4DdQzrk8/UCYO4dNxACjpp9CAi6UOhEIeq/bXMfaucRxeW8damCnKzQ1xwbD+mnVhE6VHdNHoQkZSnUDhCyzdX8OiiMv72ZjlV+2sp7t2FqeMH8ZnjB9Ktc4eEfV8RkURSKLTS3gO1PP32Fh5ZWMabZbvokBXivGP7Mm1CEScO6a7Rg4ikFIVCG1q1pZJHF5bx5Jub2V1dy9CenZk6ITx66NElt11rERE5EgqFBNh3oI5nloVHD4s37CQnyzhnVF+uO+1oRg/Q6q0ikrwUCgn27tbdPLKwjL8u2URldS0XHtePb04exuCenYMuTUTkEAqFdlJZXcPvX1nHff96n5q6ei4fP4ivnllCn655QZcmIhKlUGhn23ZXc/eLa/jz62VkZxlXnzyE6047Wk9Li0hSUCgEpGzHXm5//h1mv1VO17wcrj/9aK46aTAdO2QFXZqIZDCFQsBWllfyy+dW8893ttOnay5fO/MYPlc6kGyt0ioiAWgpFJLqbyUzu8TMfm9mj5nZ5KDraSsj+3flwasn8NiMiQzs1onvPrWMyb96hb+/vYX6+tQNZRFJPwkfKZjZA8CFwDZ3H92o/VzgDiALuM/df9boWDfgf939mpY+O1VGCo25O/NWbeOXz73DO1t307NLLrnZLWfzgMKOTBnTj/OP7adnIUSk1QK9fGRmpwJVwEMNoWBmWcC7wNnAJmARMM3dV0aO/1/gYXd/o6XPTsVQaFBX78xeupn5a3fQ0v8Cx1m+uYJ3t1aRFTJOKe7JxWP7M3lUX7rkpvTK5yISkMDnFMxsMPB0o1A4CfiBu58TeX1L5NSfRb6ed/cXmvmsGcAMgKKiohM2bNiQ2OKTxOoPKpm9tJw5S8vZvGsfudkhzhrZh08N683oAQUc3auz5ihEJCbJGAqXAee6+5cjr78AnEh49HAV4ZHDUne/t6XPTeWRwpFyd94o28nspeX8/e0t7NhzAAhvLzq8X1dG9+/KmIGFXDS2P3k5ustJRA7VUigk1fUHd78TuDPoOpKZmXHCUd054aju3DZlFOu2V7G8vIIVmytZXl7BnLfKefj1Mv7fyq3c+/njNXoQkbgEFQqbgUGNXg+MtEkcskJGSZ98Svrk8+lx4TZ356EFG7htzgq+89dl/PKy4wiFtIqriMQmqFBYBJSY2RDCYTAVuCLWN5vZFGBKcXFxgspLXWbGVZMGs3PvAX79wnt075zDd88foeW9RSQmCb+2YGaPAAuAYWa2ycyucfda4EbgOWAV8Li7r4j1M919rrvPKCjQaqTN+dqZJVx10lH8/l/vc+/L64IuR0RSRMJHCu4+rZn2Z4BnEv39M5WZcduUUezcW8PP/7Gawk45TJtQFHRZIpLkkmqiWdpWKGT872fHUFldwy1PLmP55gpuPm84+XlamE9EmpaSt6aY2RQzm1lRURF0KUmvQ3aIez9/AtP/YwiPLCzj7NtfYd6qrUGXJSJJSgviZZClG3fxnSfe5p2tu7loTH9+eMloLectkoFSZkE8SayxgwqZe9MpfOOsY3hm2RYuuec13tu6O+iyRCSJKBQyTIfsEF87q4Q/T5/I7upaLrnnNZ5dtiXoskQkSSgUMtSEId15+qZTKOmTz/UPv8EP5qxg++79QZclIgFLyVDQRHPb6FuQx2PXTuTzE4v4w4L1nPLzF7lt9nI27dwbdGkiEhBNNAsAa7dX8buX1/LkG5upd+fk4p5cevwAzhnVl04ddOeySDoJfJXURFEotL3yXft4ZGEZT725mU0791HYKYe7px3PKSU9gy5NRNqI7j6SmPUv7Mh/Tx7GK986g8dmTKRPfh5XPbiQP/47M/atEMl0CgVpUihknDi0B09cfxKnHdOL//O35Xz90TdZt70q6NJEJIFS8vJRo1VSp7/33ntBl5P26uqdX7/wLjNfWceBunrOHN6bXvm5mBlnj+zDGcN6B12iiMRBcwrSJj6s2s8Dr77P7KXl1NTVs6+mjt3VtZx2TC9+dMloBnXvFHSJIhIDhYIkxIHaeh5asJ5fv/Aexb278NRXJmnfBpEUoIlmSYgO2SG+/B9DufWCESzduIuX3tkedEki0koKBWm1y04YyMBuHfnVC++SyiNPEVEoSBvIyQrx1U+V8PamCh5dtJG6egWDSKpKyTkF3X2UfGrq6ply16us/mA3/QryKOreic652Xz73GEM79s16PJEpBFNNEu7qK6p4/mVW3n67XJ27a1hzbYqaurquf+L4xk/uHvQ5YlIhEJBArFp517+84GFbN65j7uvOJ6zR/YJuiQRQaEgAfpozwGunrWIZZt2Map/Ab3yc/nK6UczrqgbIUO3sIoEQKEggdqzv5afPbuaTTv3sry8Mrpvw9BenfnRxaOZVKzF9kTak0JBksbeA7X89Y3N7Kjaz1NvbmbDjr1cOm4A44d058zhvendNS/oEkXSnkJBklJ1TR13v7iGe19eS229U9y7C0/fdAp5OVlBlyaS1hQKktR2V9cwf+0Orv3jEo4dEJ53GNY3n29NHkYopDkHkbaWdstcaDvO9JKfl8M5o/ry3fOHA+G7ln770lqeenNzwJWJZB6NFCTp1Nc7l/52PivKKyjunc9vrjyeIT07B12WSNpIu5GCpLdQyLhr2ji+MHEw2yqr+dzvFrBm2+6gyxLJCAoFSUqDunfi+1NG8uiMiQBMnflv1n+4J+CqRNKfQkGSWkmffB6ZPpF6hy888DplO/ZqJVaRBFIoSNIr7t2FWVeP56OqA5z6y39y7R+XUF1Tx4Haeuq1IqtIm9JEs6SMVVsqeWzRRmbNXx9tG943n8evO4mueTnBFSaSYvScgqSVv7+9hfU79lBdU8c9/1xDvcPdV4zjwuP6B12aSEpoKRSy27sYkda64Lh+0V+PHVTI92ev4JYnl/Efxb0o6KQRg0hraE5BUtqZI/pw9xXj2F1dy7zVW4MuRyTlpWQo6IlmaWzMwEL6dM3l+ZUKBZHWSslQcPe57j6joKAg6FIkCYRCxvnH9uP5lVt5Yskm/rF8C6+t+VC3roocAc0pSFq44YxiZi8t55t/eSva9psrj+esEX3okJ2S//YRCYTuPpK0UbG3hvKKfbjDdX9aQtlHe+nZpQP//Obp5OuWVZEorX0kGaGgUw4j+nVlZP+u/O4LJ/DVTxXzYdUBrbYqEgeFgqSlEf268l+ThzFmYAF/mL9e8wsiMVIoSFq7atJg1m7fw2trdgRdikhKUChIWjv/2H706NyBqx5cyDHfe5bLfjtfowaRFujuI0lreTlZ3DF1HK+u+ZCyj/bwzLIP+OHTqyjomMPI/l05e2SfoEsUSSoKBUl7p5T05JSSnlRW17Bo/U4eeO19ADpkh1j6/bPp1EF/DEQaHPaWVDP7BfAjYB/wD+A44Bvu/qfEl9cy3ZIq8XJ33OHf63ZwxX2v0zEni6yQYQY/+fSxTBmjRfUk/bV2QbzJ7v5tM/s0sB64FHgFCDwUROJlFg6AiUN78K1zhvHRngMAzF5azqz56wmZAdArP5cJQ7oHWapIIGIJhYZzLgD+4u4VFvmDI5KqQiHjhjOKo6/r6p1Z89ezZMPOaNuCWz5Fv4KOQZQnEphYQuFpM1tN+PLR9WbWC6hObFki7evWC0Zw5YlFOLBuexXX/ekNvv3E2/z35GGMHVQYdHki7eawt6S6+83AJKDU3WuAvcDFiS6sJVolVdpaTlaIkj75HNMnnzNH9OGkoT3497odzIpMSotkilgmmi9t6bi7P9mmFcVBE82SSNMfWszzK7fyi8uO43Olg4IuR6TNtHbto2uA+4ErI1/3AV8CpgAXtlWRIslmxqlDAZi9VGsnSeaIZU4hBxjp7lsAzKwfMMvdr05oZSIBGz+4O5eXDuKxxRsp/u4zdMzJ4qkbJlHcOz/o0kQSJpZQGNQQCBFbgaIE1SOSVK49bSi98nPZc6CWB19bz+L1OxUKktZimVO4GygBHok0XQ6scfebElzbYWlOQdpLfb0z6rbn2FdTB8AXJw3mBxeNCrgqkSPTqofX3P3GyINrp0aaZrr7U21ZoEiyC4WMX10+lpVbKnl22RYWrNWqq5KeYl305Q1gt7u/YGadzCzf3XcnsjCRZHPu6L6cO7ove/fXcv9r7zPxJ/MA6F+Yx2PXnkROlhYdltR32FAws+nADKA7cDQwALgXODOxpYkkp6kTBrHnQB319U7ZR3tZsG4HW3ZVU9SjU9ClibRaLCOFG4AJwOsA7v6emfVOaFUiSay4dz4/vfRYAOav+ZAF63Zw69+WUdipA/0L8rj5vOFoKRhJVbGMd/e7+4GGF2aWDWiXEhFgVP8CxgwqZPPOffx73Q5+98o6tlRoFRhJXbGMFF42s+8CHc3sbOArwNzEliWSGgo65TD7hpMBmLdqK9f8YTEvrt7G0F6dgfAmP2MHFhIKaeQgqSGWULiZ8FPNy4BrgWcIP9UsIo0cFZlT+N7fln+ifdbV4zl9mK64SmpoMRTMLAt4yN2vBH7fPiWJpKbi3vn8/aunsLu6FoCdew5w/cNvUL5Ll5MkdbQYCu5eZ2ZHmVmHxvMKItK0Uf0Lor+ujjzoNmv++7z0zjYAuuRm8z+XjKZLrrYAleQUy0/mOuA1M5sD7GlodPfbE1aVSBrIy8ni4rH9eeeD3ZR9tJd9NXVs2LGXy04YyKTinkGXJ9KkZkPBzP7o7l8ALgJ+RfhOJS36IhKHO6aOi/561ZZKzrvjX1TsqwmwIpGWtTRSOMHM+gNlwF3tVI9I2iromAPAt//6Nt+fs4L8vGwev/YkenbJDbgykY+1FAr3AvOAIUDjVeeM8HMKQxNYl0ja6VeQx9fPKmFr5X62VVYzb/U21m6rUihIUmk2FNz9TuBOM/utu1/fjjWJpCUz4+tnHQPAWxt3MW/1Nqr21wZclcgnxbJKqgJBpI11yQv/0fvmX96iY04WACP6deX+L44PsiyRmFdJTTgzGwrcChS4+2VB1yOSSIN7dOa6045mR9V+AFZuqWTe6m3U17uefpZAJTQUzOwBwvs4b3P30Y3azwXuALKA+9z9Z+6+DrjGzJ5IZE0iySArZNx83vDo63tfXsuK8kr21dTRWc8wSIAS/dM3C7gbeKihIfKU9D3A2cAmYJGZzXH3lQmuRSRpde4QvoT0P3NXkpfz8TqVnxrRh9OO6RVUWZKBEhoK7v6KmQ0+qHkC4e081wGY2aPAxUBMoWBmMwjv70BRkbaKlvQwsn8BvfJzeW7lB9G2qupaVm6pVChIuwpinDoA2Njo9SbgRDPrAfwYGGdmt7j7T5t6s7vPBGZCeI/mRBcr0h5OOKobi2496xNtVz+4kO2ROQeR9pI0Fy/dfQdwXdB1iCSLvJwsqmvqgy5DMkwQobAZGNTo9cBIm4g00jEni3Xbqzjhh89/on1Y33z+PH1iQFVJugsiFBYBJWY2hHAYTAWuiOcDzGwKMKW4uDgB5Ykkh/+cNJhOuVmfaFu+uZL5a3dQU1dPTlYsGyeKxCfRt6Q+ApwO9DSzTcBt7n6/md0IPEf4ltQH3H1FPJ/r7nOBuaWlpdPbumaRZDF2UCFjBxV+ou3+V99n6cZd7NlfS2GnDgFVJuks0XcfTWum/RnCO7iJSBzyI88wVCkUJEGSZqJZRA6v4cG2z9/3Oh2yP758FDLjW+cM48wRfYIqTdJESoaC5hQkU00Y0p1PjxsQ3dWtwQurtvLqmg8VCtJqKRkKmlOQTNUrP5dfXT72kPbxP35Bt69Km9DtCyJpoGNO1iGjB5EjkZIjBRH5pI45WWytrGb55opDjg0o7Ei3zpqUltgoFETSQEGnHOav3cGFd716yLGS3l14/r9OC6AqSUUpGQqaaBb5pNs/N4aV5ZWHtD/8ehnLmhg9iDQnJUNBE80inzSwWycGdut0SPuCdTtYsmFnABVJqtJEs0ga65Ad4kCd7kqS2CkURNJYblaIA7X1uGuVeYlNSl4+EpHYNDz1fObtL9PUzs+FnTrwhy9NoIu2AJWIlPxJ0ESzSGzOGdWX97ZVUVt/6Ejhg4pqlmzYycaP9jKiX9cAqpNklJKhoIlmkdiU9Mnnjqnjmjz2/MqtTH9oMXVNBIZkLs0piGSo7KzwBaUaTURLIwoFkQyVHQqHQlOXliRzKRREMlR2KPzHv7ZOoSAfS8k5BRFpvZzI5aPHFpWxYO2HTZ5zSkkvJgzp3p5lScBSMhR095FI6/Uv7Eh+Xjaz3ypv8rh7+Inov1w3qZ0rkyBZKj/UUlpa6osXLw66DJG0dNUDC9m1r4bZN5wcdCnSxsxsibuXNnVMcwoi0qTskFGrO5MyjkJBRJqUFTI9w5CBFAoi0qScrJCeYchACgURaZJGCplJoSAiTcrOMj3YloFS8pZUEUm87JCxrXI/n713/mHPNYxrTxvKmSP6tENlkkgpOVIwsylmNrOiQtsMiiTKeaP7MX5IN3KyQof9enPjTl5YtTXokqUNpORIQaukiiTeGcN7c8bw3jGdO/En8zT/kCZScqQgIsklPCkddBXSFhQKItJqoRDUp/DqCPIxhYKItFqW6U6ldKFQEJFWywoZ9QqFtKBQEJFW04Nu6UOhICKtFjKjTnMKaUGhICKtppFC+kjJ5xREJLlkh4xF6z/i0t+8Fvd78/NyuOuKcXTNy0lAZRKvlAwF7bwmklw+WzqI51Z8EPf7dlQd4I2y7by/fQ9jBhUmoDKJl3ZeE5HA/HP1Nq6etYinvjKJcUXdgi4nY2jnNRFJSmbh/2o6InkoFEQkMKFIKqTyFYt0o1AQkcBkhcKhoJFC8lAoiEhgPr58pFRIFgoFEQlMw+UjhULyUCiISGA+nlMIuBCJUiiISGBCunyUdBQKIhIYi4wUtERG8lAoiEhgGu4+0kAheSgURCQwunyUfBQKIhKYj+8+CrgQiUrJBfFEJD00PKdw14vv8ejCsoR8j95d8/jxJaMJNQxLpEUpGQpaJVUkPRzVozMnF/egYl8NW3dXt/nnf1R1gHmrt/GNs0ro3TWvzT8/HaVkKLj7XGBuaWnp9KBrEZEj1yU3m4e/PDFhn//w6xu49anl6OpU7DSnICJpy9DdTfFSKIhI2mqYs3CNFWKmUBCRtNUwt6yRQuwUCiKSthouH+k5iNgpFEQkfWmkEDeFgoikLT2ZED+FgoikLS3NHT+FgoikLe3sFj+FgoikrY9vSZVYKRREJG19/PCaYiFWCgURSVsaKcRPoSAiactMI4V4KRREJG013JKqTIidQkFE0pYuH8VPoSAiaUvPKcRPoSAiaavh8pGeU4idQkFE0pZp7aO4KRREJI1FLh9pViFmCgURSVvaTyF+SbNHs5l1Bn4DHABecveHAy5JRFKcaaI5bgkdKZjZA2a2zcyWH9R+rpm9Y2ZrzOzmSPOlwBPuPh24KJF1iUhmiD6noMtHMUv0SGEWcDfwUEODmWUB9wBnA5uARWY2BxgILIucVpfgukQkAzRMNN/+/Lt079Qh2GLa2BUnFlE6uHubf25CQ8HdXzGzwQc1TwDWuPs6ADN7FLiYcEAMBJbSwgjGzGYAMwCKioravmgRSRslvfMZ1ieftdurWBt0MW1s8qi+CfncIOYUBgAbG73eBJwI3AncbWYXAHObe7O7zwRmApSWlmpMKCLNKurRiee+cWrQZaSUpJlodvc9wNVB1yEiksmCuCV1MzCo0euBkTYREQlYEKGwCCgxsyFm1gGYCsyJ5wPMbIqZzayoqEhIgSIimSrRt6Q+AiwAhpnZJjO7xt1rgRuB54BVwOPuviKez3X3ue4+o6CgoO2LFhHJYIm++2haM+3PAM8k8nuLiEj8tMyFiIhEpWQoaE5BRCQxUjIUNKcgIpIYlsobWpvZdmBDo6YCoCLG1z2BDxNQ1sHfsy3Ob+mc5o7F0xcHv870vjm4LZP7pqm25voGEtM/6pvmxds3De8pdPdeTR5197T5AmbG+hpY3B41tMX5LZ3T3LF4+kJ9c+h5B/VHxvZNDD8rBx9r8/5R37Rd38TynpS8fNSCg5fHONzr9qihLc5v6ZzmjsXbF+qb5tsyuW+aalPfNN+W7H1z2Pek9OWj1jCzxe5eGnQdyUh90zz1TcvUP81Llb5Jt5FCPGYGXUASU980T33TMvVP81KibzJ2pCAiIofK5JGCiIgcRKEgIiJRCgUREYlSKESYWWcz+4OZ/d7Mrgy6nmRiZkPN7H4zeyLoWpKNmV0S+Zl5zMwmB11PMjGzEWZ2r5k9YWbXB11Pson8nbPYzC4MupbG0joUzOwBM9tmZssPaj/XzN4xszVmdnOk+VLgCXefDlzU7sW2s3j6xt3Xufs1wVTa/uLsm79FfmauAy4Pot72FGffrHL364DPAScHUW97ivPvG4DvAI+3b5WHl9ahAMwCzm3cYGZZwD3AecBIYJqZjSS8A1zD3tF17VhjUGYRe99kmlnE3zffixxPd7OIo2/M7CLg72TGUvmziLFvzOxsYCWwrb2LPJy0DgV3fwX46KDmCcCayL9+DwCPAhcDmwgHA6R5v0DcfZNR4ukbC/s58Ky7v9Hetba3eH9u3H2Ou58HpP0l2Tj75nRgInAFMN3MkubvnIRuspOkBvDxiADCYXAicCdwt5ldQPs8np6MmuwbM+sB/BgYZ2a3uPtPA6kuWM393NwEnAUUmFmxu98bRHEBa+7n5nTCl2VzyYyRQlOa7Bt3vxHAzL4IfOju9QHU1qRMDIUmufse4Oqg60hG7r6D8DVzOYi730n4HxRyEHd/CXgp4DKSmrvPCrqGgyXNkKUdbQYGNXo9MNIm6puWqG+ap75pXsr1TSaGwiKgxMyGmFkHYCowJ+CakoX6pnnqm+apb5qXcn2T1qFgZo8AC4BhZrbJzK5x91rgRuA5YBXwuLuvCLLOIKhvmqe+aZ76pnnp0jdaEE9ERKLSeqQgIiLxUSiIiEiUQkFERKIUCiIiEqVQEBGRKIWCiIhEKRREmmBmhWb2lcOcc7qZPd1eNYm0B4WCSNMKgRZDQSQdKRREmvYz4GgzW2pmv4x8LTezZWZ2yGY6ZjbezN40s6MjO2o9YGYLI20XR875opk9aWb/MLP3zOwXkfYsM5vV6PO/0c6/V5EorZIq0rSbgdHuPtbMPkN4ldgxQE9gkZm90nCimU0C7gIudvcyM/sJ8KK7f8nMCoGFZvZC5PSxwDhgP/COmd0F9AYGuPvoyOcVttPvUeQQGimIHN4pwCPuXufuW4GXgfGRYyOAmcAUdy+LtE0GbjazpYSXjs4DiiLH5rl7hbtXE9556yhgHTDUzO4ys3OByvb4TYk0RaEg0jpbgGrC//pvYMBn3H1s5KvI3VdFju1vdF4dkO3uOwmPQl4iPCK5L/FlizRNoSDStN1AfuTX/wIuj1z77wWcCiyMHNsFXAD8NLLTGIRXxLzJzAzAzBoHxiHMrCcQcve/Et7r+fi2/I2IxENzCiJNcPcdZvaamS0HngXeBt4CHPi2u39gZsMj5241swuBZ83sS8APgV8Db0f23n0fuLCFbzcAeLDRPr23JOZ3JXJ4WjpbRESidPlIRESiFAoiIhKlUBARkSiFgoiIRCkUREQkSqEgIiJRCgUREYlSKIiISNT/B8TMH38Jv08yAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "freqs = [freq for token, freq in bigram_vocab.token_freqs]\n",
    "axes = plt.gca()\n",
    "axes.plot(range(len(freqs)), freqs)\n",
    "axes.set_xlabel('tokens')\n",
    "axes.set_ylabel('freqs')\n",
    "axes.set_xscale('log')\n",
    "axes.set_yscale('log')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}