transformer.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.7"
    },
    "colab": {
      "name": "transformer.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dnQL8znm6EGb",
        "colab_type": "text"
      },
      "source": [
        "# Transformer in PyTorch\n",
        "\n",
        "This is a pytorch implementation of the Transformer, which is also my note.\n",
        "\n",
        "<img src=\"https://i.imgur.com/thaMgTE.png\"/>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kKaaXZB06EGf",
        "colab_type": "text"
      },
      "source": [
        "## Import Dependencies"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pCsdXr496EGh",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 258
        },
        "outputId": "bd915613-7533-443c-b1c6-78e7f81a465e"
      },
      "source": [
        "!pip3 install --upgrade torch torchvision torchtext\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.optim as optim\n",
        "import torch.nn.functional as F\n",
        "from torch.autograd import Variable\n",
        "\n",
        "import spacy\n",
        "\n",
        "import random\n",
        "import math\n",
        "import os\n",
        "import time\n",
        "from tqdm import tqdm\n",
        "\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already up-to-date: torch in /usr/local/lib/python3.6/dist-packages (1.3.1)\n",
            "Requirement already up-to-date: torchvision in /usr/local/lib/python3.6/dist-packages (0.4.2)\n",
            "Requirement already up-to-date: torchtext in /usr/local/lib/python3.6/dist-packages (0.4.0)\n",
            "Requirement already satisfied, skipping upgrade: numpy in /usr/local/lib/python3.6/dist-packages (from torch) (1.17.4)\n",
            "Requirement already satisfied, skipping upgrade: pillow>=4.1.1 in /usr/local/lib/python3.6/dist-packages (from torchvision) (4.3.0)\n",
            "Requirement already satisfied, skipping upgrade: six in /usr/local/lib/python3.6/dist-packages (from torchvision) (1.12.0)\n",
            "Requirement already satisfied, skipping upgrade: tqdm in /usr/local/lib/python3.6/dist-packages (from torchtext) (4.28.1)\n",
            "Requirement already satisfied, skipping upgrade: requests in /usr/local/lib/python3.6/dist-packages (from torchtext) (2.21.0)\n",
            "Requirement already satisfied, skipping upgrade: olefile in /usr/local/lib/python3.6/dist-packages (from pillow>=4.1.1->torchvision) (0.46)\n",
            "Requirement already satisfied, skipping upgrade: urllib3<1.25,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->torchtext) (1.24.3)\n",
            "Requirement already satisfied, skipping upgrade: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->torchtext) (2019.9.11)\n",
            "Requirement already satisfied, skipping upgrade: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->torchtext) (3.0.4)\n",
            "Requirement already satisfied, skipping upgrade: idna<2.9,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->torchtext) (2.8)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9HSouO9mloLX",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "354ee7e4-d31a-461d-841f-1aa61ab404fc"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "scrolled": true,
        "id": "0HRHiId66EGn",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "116f8d0a-52ea-4fba-f4d3-40de41306940"
      },
      "source": [
        "device_num = '0'\n",
        "device = torch.device('cuda:'+device_num if torch.cuda.is_available() else 'cpu')\n",
        "print('device type:', device)"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "device type: cuda:0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cd0de8lK6EGs",
        "colab_type": "text"
      },
      "source": [
        "## Prepare for Dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TpaHQsCU6EGt",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 275
        },
        "outputId": "a217ef0b-2bbe-4226-e45a-614230a40148"
      },
      "source": [
        "import torchtext\n",
        "from torchtext.datasets import TranslationDataset, Multi30k, IWSLT\n",
        "from torchtext import data\n",
        "\n",
        "!python -m spacy download de\n",
        "!python -m spacy download en\n",
        "spacy_de = spacy.load('de')\n",
        "spacy_en = spacy.load('en')"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: de_core_news_sm==2.1.0 from https://github.com/explosion/spacy-models/releases/download/de_core_news_sm-2.1.0/de_core_news_sm-2.1.0.tar.gz#egg=de_core_news_sm==2.1.0 in /usr/local/lib/python3.6/dist-packages (2.1.0)\n",
            "\u001b[38;5;2m✔ Download and installation successful\u001b[0m\n",
            "You can now load the model via spacy.load('de_core_news_sm')\n",
            "\u001b[38;5;2m✔ Linking successful\u001b[0m\n",
            "/usr/local/lib/python3.6/dist-packages/de_core_news_sm -->\n",
            "/usr/local/lib/python3.6/dist-packages/spacy/data/de\n",
            "You can now load the model via spacy.load('de')\n",
            "Requirement already satisfied: en_core_web_sm==2.1.0 from https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-2.1.0/en_core_web_sm-2.1.0.tar.gz#egg=en_core_web_sm==2.1.0 in /usr/local/lib/python3.6/dist-packages (2.1.0)\n",
            "\u001b[38;5;2m✔ Download and installation successful\u001b[0m\n",
            "You can now load the model via spacy.load('en_core_web_sm')\n",
            "\u001b[38;5;2m✔ Linking successful\u001b[0m\n",
            "/usr/local/lib/python3.6/dist-packages/en_core_web_sm -->\n",
            "/usr/local/lib/python3.6/dist-packages/spacy/data/en\n",
            "You can now load the model via spacy.load('en')\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rovOKwZk6EGx",
        "colab_type": "text"
      },
      "source": [
        "Define toeknizers for both languages."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZaO66tER6EGy",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Tokenizes German\n",
        "def tokenize_de(text):\n",
        "    return [tok.text for tok in spacy_de.tokenizer(text)]\n",
        "\n",
        "# Tokenizes English\n",
        "def tokenize_en(text):\n",
        "    return [tok.text for tok in spacy_en.tokenizer(text)]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TEsEI2XM6EG2",
        "colab_type": "text"
      },
      "source": [
        "Preprocess them as source and target. ```<sos>``` stands for Start Of Sentence and ```<eos>``` for End Of Sentence. Set ```batch_first``` to true so batch_size would be the first dimension."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aAaJThhN6EG3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# source (Gernam)\n",
        "GER = data.Field(tokenize=tokenize_de, init_token='<sos>', eos_token='<eos>', lower=True, batch_first=True)\n",
        "\n",
        "# target (English)\n",
        "ENG = data.Field(tokenize=tokenize_en, init_token='<sos>', eos_token='<eos>', lower=True, batch_first=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xEdo3euP6EG6",
        "colab_type": "text"
      },
      "source": [
        "Split into train and test set, and build the vocab."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pt7fVOOz6EG7",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "MAX_LEN = 100\n",
        "MIN_FREQ = 2\n",
        "\n",
        "train_data, valid_data, test_data = Multi30k.splits(\n",
        "    exts=('.de', '.en'), \n",
        "    fields=(GER, ENG), \n",
        "    filter_pred=lambda x: len(vars(x)['src']) <= MAX_LEN and len(vars(x)['trg']) <= MAX_LEN\n",
        ")\n",
        "\n",
        "GER.build_vocab(train_data, min_freq=MIN_FREQ)\n",
        "ENG.build_vocab(train_data, min_freq=MIN_FREQ)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nmcdZG4k6EG_",
        "colab_type": "text"
      },
      "source": [
        "Batchify the data."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "x_2Q4bHJ6EHA",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "BATCH_SIZE = 128\n",
        "\n",
        "train_iterator = data.BucketIterator(\n",
        "    dataset=train_data, \n",
        "    batch_size=BATCH_SIZE, \n",
        "    shuffle=True, \n",
        "    sort_key=lambda x: data.interleave_keys(len(x.src), len(x.trg)), \n",
        "    device=device\n",
        ")\n",
        "\n",
        "valid_iterator = data.BucketIterator(\n",
        "    dataset=valid_data, \n",
        "    batch_size=BATCH_SIZE, \n",
        "    device=device\n",
        ")\n",
        "\n",
        "test_iterator = data.BucketIterator(\n",
        "    dataset=test_data, \n",
        "    batch_size=BATCH_SIZE, \n",
        "    device=device\n",
        ")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "Sh7me_vA7VVo"
      },
      "source": [
        "## Transformer Model"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kE92AzNj6EHE",
        "colab_type": "text"
      },
      "source": [
        "### Multi-Head Attention (SubLayer)\n",
        "\n",
        "The following image illustrates a single attention head and Multi-Head attention.\n",
        "\n",
        "<img src=\"https://i.imgur.com/ScaPfNh.png\"/>\n",
        "\n",
        "and the Scaled Dot-Product attention is represented by the formula:\n",
        "\n",
        "<img src=\"https://i.imgur.com/vrnyrgS.png\" width=\"550\"/>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0KyACQOT6EHF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class ScaledDotProductAttention(nn.Module):\n",
        "    def __init__(self, scaled_term, attn_dropout=0.1):\n",
        "        super(ScaledDotProductAttention, self).__init__()\n",
        "        self.scaled_term = scaled_term\n",
        "        self.dropout = nn.Dropout(attn_dropout)\n",
        "        self.softmax = nn.Softmax(dim=-1)\n",
        "\n",
        "    def forward(self, q, k, v, mask=None):\n",
        "        # Score\n",
        "        attn = torch.matmul(q, k) # [B, n_head, T, T]\n",
        "        attn = attn / self.scaled_term\n",
        "\n",
        "        if mask is not None:\n",
        "            attn = attn.masked_fill(mask==0, -1e10)\n",
        "\n",
        "        attn = self.softmax(attn) # [B, n_head, T, T]\n",
        "        attn = self.dropout(attn)\n",
        "        \n",
        "        # Sum\n",
        "        output = torch.matmul(attn, v) # [B, n_head, T, H//n_head]\n",
        "\n",
        "        return output, attn\n",
        "\n",
        "\n",
        "class MultiHeadAttention(nn.Module):\n",
        "    def __init__(self, hidden_size, n_head, device, dropout=0.1):\n",
        "        super(MultiHeadAttention, self).__init__()\n",
        "        self.hidden_size = hidden_size\n",
        "        self.n_head = n_head\n",
        "\n",
        "        self.w_q = nn.Linear(hidden_size, hidden_size)\n",
        "        self.w_k = nn.Linear(hidden_size, hidden_size)\n",
        "        self.w_v = nn.Linear(hidden_size, hidden_size)\n",
        "        self.scaled_dot_attention = ScaledDotProductAttention(torch.sqrt(torch.FloatTensor([hidden_size//n_head])).to(device))\n",
        "        self.dropout = nn.Dropout(dropout)\n",
        "        self.output_layer = nn.Linear(hidden_size, hidden_size)\n",
        "        self.device = device\n",
        "\n",
        "    def forward(self, q, k, v, mask=None):\n",
        "        batch_size = q.shape[0]\n",
        "        '''\n",
        "        query = key = value: [B, T, H]\n",
        "        mask: [B, T, 1]\n",
        "        '''\n",
        "        # Project and split\n",
        "        q = self.w_q(q).view(batch_size, -1, self.n_head, self.hidden_size//self.n_head) # [B, T, H] -> [B, T, n_head, H//n_head]\n",
        "        k = self.w_k(k).view(batch_size, -1, self.n_head, self.hidden_size//self.n_head) # [B, T, H] -> [B, T, n_head, H//n_head]\n",
        "        v = self.w_v(v).view(batch_size, -1, self.n_head, self.hidden_size//self.n_head) # [B, T, H] -> [B, T, n_head, H//n_head]\n",
        "        \n",
        "        q = q.permute(0, 2, 1, 3) # [B, n_head, T, H//n_head]\n",
        "        k = k.permute(0, 2, 3, 1) # [B, n_head, H//n_head, T]\n",
        "        v = v.permute(0, 2, 1, 3) # [B, n_head, T, H//n_head]\n",
        "\n",
        "        output, attn = self.scaled_dot_attention(q, k, v, mask) # [B, n_head, T, T], [B, n_head, T, H//n_head]\n",
        "        output = output.transpose(1, 2).contiguous() # [B, T, n_head, H//n_head]\n",
        "        output = output.view(batch_size, -1, self.n_head * (self.hidden_size//self.n_head)) # [B, T, H]\n",
        "        \n",
        "        output = self.output_layer(output)\n",
        "        output = self.dropout(output)\n",
        "        \n",
        "        # return output, attn\n",
        "        return output"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JtVszmUc6EHI",
        "colab_type": "text"
      },
      "source": [
        "### Encoder Layer\n",
        "\n",
        "<img src=\"https://i.imgur.com/dZwWje2.png\" />\n",
        "\n",
        "#### Post-LN or Pre-LN\n",
        "\n",
        "You can find different version of Transformer implementation that use either Post-LayerNormalization or Pre-LayerNormalization on github. Here we use the flag ```pre_lnorm``` to decide whether it is Pre-LN or not.\n",
        "\n",
        "According to the official implementation note, whether there's a Pre-LN or not, we must apply LayerNorm to the last output of both Encoder and Decoder, since the output can grow very large, being the sum of a whole stack of unnormalized lauer outputs.\n",
        "\n",
        "<img src=\"https://i.imgur.com/s7TkryI.png\"/>\n",
        "\n",
        "And here's the table for detail:\n",
        "\n",
        "<img src=\"https://i.imgur.com/J1bx9yC.png\" />\n",
        "\n",
        "As my experiment results, Pre-LN converge much faster than Post-LN, and Pre-LN doesn't even need warm-up steps."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JN0X1rl36EHJ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class PositionwiseFeedForward(nn.Module):\n",
        "    def __init__(self, hidden_size, filter_size, dropout):\n",
        "        super(PositionwiseFeedForward, self).__init__()\n",
        "        self.pff = nn.Sequential(\n",
        "            nn.Linear(hidden_size, filter_size), \n",
        "            nn.ReLU(inplace=True), \n",
        "            nn.Dropout(dropout), \n",
        "            nn.Linear(filter_size, hidden_size), \n",
        "            nn.Dropout(dropout)\n",
        "        )\n",
        "    \n",
        "    def forward(self, src):\n",
        "        src = self.pff(src)\n",
        "\n",
        "        return src\n",
        "        \n",
        "    \n",
        "class EncoderLayer(nn.Module):\n",
        "    def __init__(self, hidden_size, filter_size, n_head, pre_lnorm, device, dropout):\n",
        "        super(EncoderLayer, self).__init__()\n",
        "        # self-attention part\n",
        "        self.self_attn = MultiHeadAttention(hidden_size, n_head, device)\n",
        "        self.self_attn_norm = nn.LayerNorm(hidden_size)\n",
        "        \n",
        "        # feed forward network part\n",
        "        self.pff = PositionwiseFeedForward(hidden_size, filter_size, dropout)\n",
        "        self.pff_norm = nn.LayerNorm(hidden_size)\n",
        "\n",
        "        self.pre_lnorm = pre_lnorm\n",
        "        \n",
        "    def forward(self, src, src_mask):\n",
        "        if self.pre_lnorm:\n",
        "            pre = self.self_attn_norm(src)\n",
        "            src = src + self.self_attn(pre, pre, pre, src_mask) # residual connection\n",
        "\n",
        "            pre = self.pff_norm(src)\n",
        "            src = src + self.pff(pre) # residual connection\n",
        "        else:\n",
        "            src = self.self_attn_norm(src + self.self_attn(src, src, src, src_mask)) # residual connection + layerNorm\n",
        "            src = self.pff_norm(src + self.pff(src)) # residual connection + layerNorm\n",
        "        \n",
        "        return src"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5KlIeyUM6EHM",
        "colab_type": "text"
      },
      "source": [
        "### Encoder\n",
        "\n",
        "Encoder consists of multiple layers of EncoderLayer. And also remember the last LayerNorm metioned previously.\n",
        "\n",
        "#### Embeddings: Token Embeddings (wte) + Positional Encoding (wpe)\n",
        "\n",
        "<img src=\"https://i.imgur.com/PWBhjhL.png\" />\n",
        "\n",
        "We may see lots of ways to implement positional encoding. Here are some of the common ways:\n",
        "\n",
        "1. nn.Embedding(...).\n",
        "    - The weight could be previously defined, or simply a fresh new embeddings.\n",
        "    - It could be trainable or un-trainable. The idea of the paper is to use a fixed table.\n",
        "\n",
        "2. A class instance.\n",
        "    - Like [huggingface's transformer](https://github.com/huggingface/transformers/blob/master/transformers/modeling_transfo_xl.py).\n",
        "\n",
        "As what I've tested, they could achieve similar result, with nn.Embedding() slighly better. Using fresh new trainable nn.Embedding() or loading the pretrained weight doesn't make much differnce. I leave all the implementations here as a note."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nv4yCMITUR5P",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class PositionalEmbedding(nn.Module):\n",
        "    def __init__(self, embed_size):\n",
        "        super(PositionalEmbedding, self).__init__()\n",
        "        self.embed_size = embed_size # hidden_size\n",
        "\n",
        "        inv_timescales = 1 / (10000 ** (torch.arange(0.0, embed_size, 2.0) / embed_size))\n",
        "        self.register_buffer('inv_timescales', inv_timescales)\n",
        "\n",
        "    def forward(self, pos):\n",
        "        scaled_time = torch.ger(pos, self.inv_timescales) # [T, H//2]\n",
        "        pos_embed = torch.cat([torch.sin(scaled_time), torch.cos(scaled_time)], dim=-1) # [T, H]\n",
        "        \n",
        "        return pos_embed[None, :, :]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "W4KozZZoXIMs",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class PositionalEncoding(nn.Module):\n",
        "    \"Implement the PE function.\"\n",
        "    def __init__(self, d_model, max_len=5000):\n",
        "        super(PositionalEncoding, self).__init__()\n",
        "        \n",
        "        # Compute the positional encodings once in log space.\n",
        "        pe = torch.zeros(max_len, d_model)\n",
        "        position = torch.arange(0, max_len).unsqueeze(1)\n",
        "        div_term = torch.exp(torch.arange(0, d_model, 2) *\n",
        "                             -(math.log(10000.0) / d_model))\n",
        "        pe[:, 0::2] = torch.sin(position * div_term)\n",
        "        pe[:, 1::2] = torch.cos(position * div_term)\n",
        "        pe = pe.unsqueeze(0)\n",
        "        self.register_buffer('pe', pe)\n",
        "        \n",
        "    def forward(self, x):\n",
        "        x = Variable(self.pe[:, :x.size(1)], requires_grad=False)\n",
        "        return x"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wcs1u_HlUR1q",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def positional_encoding_table(n_position, d_hid, padding_idx=None):\n",
        "    ''' Sinusoid position encoding table '''\n",
        "\n",
        "    def cal_angle(position, hid_idx):\n",
        "        return position / np.power(10000, 2 * (hid_idx // 2) / d_hid)\n",
        "\n",
        "    def get_posi_angle_vec(position):\n",
        "        return [cal_angle(position, hid_j) for hid_j in range(d_hid)]\n",
        "\n",
        "    sinusoid_table = np.array([get_posi_angle_vec(pos_i) for pos_i in range(n_position)])\n",
        "\n",
        "    sinusoid_table[:, 0::2] = np.sin(sinusoid_table[:, 0::2])  # dim 2i\n",
        "    sinusoid_table[:, 1::2] = np.cos(sinusoid_table[:, 1::2])  # dim 2i+1\n",
        "\n",
        "    if padding_idx is not None:\n",
        "        # zero vector for padding dimension\n",
        "        sinusoid_table[padding_idx] = 0.\n",
        "\n",
        "    return torch.FloatTensor(sinusoid_table)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LrOWTjFx6EHN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class Encoder(nn.Module):\n",
        "    def __init__(self, input_size, hidden_size, filter_size, n_head, dropout, n_layers, pre_lnorm, device):\n",
        "        super(Encoder, self).__init__()\n",
        "        self.hidden_size = hidden_size\n",
        "        self.embed_scale = hidden_size ** 0.5\n",
        "        self.wte = nn.Embedding(input_size, hidden_size) # token embeddings\n",
        "        # self.wpe = PositionalEmbedding(hidden_size) # positional embeddings\n",
        "        # self.wpe = nn.Embedding(1000, hidden_size)\n",
        "        # self.wpe = PositionalEncoding(hidden_size)\n",
        "        max_len = 1000\n",
        "        self.wpe = nn.Embedding.from_pretrained(positional_encoding_table(max_len+1, hidden_size, padding_idx=GER.vocab.stoi['<pad>']), freeze=True)\n",
        "        self.embed_dropout = nn.Dropout(dropout)\n",
        "        self.layers = nn.ModuleList([EncoderLayer(hidden_size, filter_size, n_head, pre_lnorm, device, dropout) \n",
        "                                     for _ in range(n_layers)])\n",
        "        self.pre_lnorm = pre_lnorm\n",
        "        self.last_norm = nn.LayerNorm(hidden_size)\n",
        "        self.device = device\n",
        "        \n",
        "    def forward(self, src, src_mask):\n",
        "        # token embedding + positional encoding\n",
        "        # pos = torch.arange(src.shape[1], dtype=torch.float32).to(self.device)\n",
        "        pos = torch.arange(0, src.shape[1]).unsqueeze(0).repeat(src.shape[0], 1).to(self.device)\n",
        "        src = self.wte(src) * self.embed_scale + self.wpe(pos) # [B, T, H]\n",
        "        src = self.embed_dropout(src)\n",
        "        \n",
        "        for layer in self.layers:\n",
        "            src = layer(src, src_mask)\n",
        "        \n",
        "        if self.pre_lnorm:\n",
        "            src = self.last_norm(src)\n",
        "        \n",
        "        return src"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RTGIdmny6EHR",
        "colab_type": "text"
      },
      "source": [
        "### Decoder Layer\n",
        "\n",
        "<img src=\"https://i.imgur.com/y4ZE8UL.png\" />\n",
        "\n",
        "Target masking is a crutial part in decoder. It masked out the rest of the sequence after current time step, so the model won't peek the answer. Otherwise, we may find low training perplexity but suck at testing."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "f0CAth8p6EHS",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class DecoderLayer(nn.Module):\n",
        "    def __init__(self, hidden_size, filter_size, n_head, pre_lnorm, device, dropout):\n",
        "        super(DecoderLayer, self).__init__()\n",
        "        # self-attention part\n",
        "        self.self_attn = MultiHeadAttention(hidden_size, n_head, device)\n",
        "        self.self_attn_norm = nn.LayerNorm(hidden_size)\n",
        "        \n",
        "        # encoder-to-decoder self-attention part\n",
        "        self.ed_self_attn = MultiHeadAttention(hidden_size, n_head, device)\n",
        "        self.ed_self_attn_norm = nn.LayerNorm(hidden_size)\n",
        "        \n",
        "        # feed forward network part\n",
        "        self.pff = PositionwiseFeedForward(hidden_size, filter_size, dropout)\n",
        "        self.pff_norm = nn.LayerNorm(hidden_size)\n",
        "\n",
        "        self.pre_lnorm = pre_lnorm\n",
        "        \n",
        "    def forward(self, enc_out, enc_out_mask, trg, trg_mask):\n",
        "        if self.pre_lnorm:\n",
        "            ris = self.self_attn_norm(trg)\n",
        "            trg = trg + self.self_attn(ris, ris, ris, trg_mask)\n",
        "\n",
        "            ris = self.ed_self_attn_norm(trg)\n",
        "            trg = trg + self.ed_self_attn(ris, enc_out, enc_out, enc_out_mask)\n",
        "\n",
        "            ris = self.pff_norm(trg)\n",
        "            trg = trg + self.pff(ris)\n",
        "        else:\n",
        "            trg = self.self_attn_norm(trg + self.self_attn(trg, trg, trg, trg_mask))\n",
        "            trg = self.ed_self_attn_norm(trg + self.ed_self_attn(trg, enc_out, enc_out, enc_out_mask))\n",
        "            trg = self.pff_norm(trg + self.pff(trg))\n",
        "            \n",
        "        return trg"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3YtfdpvQ6EHV",
        "colab_type": "text"
      },
      "source": [
        "### Decoder"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "d3jYmAxX6EHW",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class Decoder(nn.Module):\n",
        "    def __init__(self, input_size, hidden_size, filter_size, n_head, dropout, n_layers, pre_lnorm, device):\n",
        "        super(Decoder, self).__init__()\n",
        "        self.hidden_size = hidden_size\n",
        "        self.dropout = dropout\n",
        "        self.embed_scale = hidden_size ** 0.5\n",
        "        self.wte = nn.Embedding(input_size, hidden_size) # token embeddings\n",
        "        # self.wpe = PositionalEmbedding(hidden_size) # positional embeddings\n",
        "        # self.wpe = nn.Embedding(1000, hidden_size)\n",
        "        # self.wpe = PositionalEncoding(hidden_size)\n",
        "        max_len = 1000\n",
        "        self.wpe = nn.Embedding.from_pretrained(positional_encoding_table(max_len+1, hidden_size, padding_idx=ENG.vocab.stoi['<pad>']), freeze=True)\n",
        "        self.embed_dropout = nn.Dropout(dropout)\n",
        "        self.layers = nn.ModuleList([DecoderLayer(hidden_size, filter_size, n_head, pre_lnorm, device, dropout)\n",
        "                                     for _ in range(n_layers)])\n",
        "        self.pre_lnorm = pre_lnorm\n",
        "        self.last_norm = nn.LayerNorm(hidden_size)\n",
        "        self.device = device\n",
        "        \n",
        "    def forward(self, enc_out, enc_out_mask, trg, trg_mask):\n",
        "        # token embedding + positional encoding\n",
        "        # pos = torch.arange(trg.shape[1], dtype=torch.float32).to(self.device)\n",
        "        pos = torch.arange(0, trg.shape[1]).unsqueeze(0).repeat(trg.shape[0], 1).to(self.device)\n",
        "        trg = self.wte(trg) * self.embed_scale + self.wpe(pos) # [B, T, H]\n",
        "        trg = self.embed_dropout(trg)\n",
        "        \n",
        "        for layer in self.layers:\n",
        "            trg = layer(enc_out, enc_out_mask, trg, trg_mask)\n",
        "        \n",
        "        if self.pre_lnorm:\n",
        "            trg = self.last_norm(trg)\n",
        "        \n",
        "        return trg"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RN2n3Gyd6EHZ",
        "colab_type": "text"
      },
      "source": [
        "### Transformer\n",
        "\n",
        "Now put all the layers together."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DCca7a0Q6EHa",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class Transformer(nn.Module):\n",
        "    def __init__(self, enc_input_size, dec_input_size, hidden_size, filter_size, n_head, dropout, n_layers, pre_lnorm, device, maxlen=50):\n",
        "        super(Transformer, self).__init__()\n",
        "        self.encoder = Encoder(enc_input_size, hidden_size, filter_size, n_head, dropout, n_layers, pre_lnorm, device)\n",
        "        self.decoder = Decoder(dec_input_size, hidden_size, filter_size, n_head, dropout, n_layers, pre_lnorm, device)\n",
        "        self.project = nn.Linear(hidden_size, dec_input_size)\n",
        "        self.maxlen = maxlen\n",
        "        self.src_sos_idx = GER.vocab.stoi['<sos>']\n",
        "        self.src_eos_idx = GER.vocab.stoi['<eos>']\n",
        "        \n",
        "    def forward(self, src, src_mask, trg, trg_mask):\n",
        "        enc_out = self.encoder(src, src_mask)\n",
        "        dec_out = self.decoder(enc_out, src_mask, trg, trg_mask)\n",
        "        output = self.project(dec_out)\n",
        "        \n",
        "        return output\n",
        "        \n",
        "    def inference(self, src):\n",
        "        batch_size, src_len = src.shape\n",
        "        trg = src.new_full((batch_size, 1), self.src_sos_idx)\n",
        "        src_mask, trg_mask = make_masks(src, trg)\n",
        "        \n",
        "        enc_out = self.encoder(src, src_mask)\n",
        "        \n",
        "        for t in range(self.maxlen-1):\n",
        "            dec_out = self.decoder(enc_out, src_mask, trg, trg_mask) # [B, T, H]\n",
        "            output = self.project(dec_out) # [B, vocab_size]\n",
        "            output = torch.argmax(output[:, -1], dim=1) # [B]\n",
        "            output = output.unsqueeze(1) # [B, 1]\n",
        "            trg = torch.cat((trg, output), dim=1)\n",
        "            src_mask, trg_mask = make_masks(src, trg)\n",
        "            \n",
        "        return trg\n",
        "        "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZARqYuK66EHe",
        "colab_type": "text"
      },
      "source": [
        "## Train the model"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CZGUS5UWgjPm",
        "colab_type": "text"
      },
      "source": [
        "### Masking"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3XsToQlxgcv6",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# helper function to make mask for source and target.\n",
        "def make_masks(src, trg):\n",
        "    src_mask = (src != GER.vocab.stoi['<pad>']).unsqueeze(1).unsqueeze(2)\n",
        "    trg_pad_mask = (trg != GER.vocab.stoi['<pad>']).unsqueeze(1).unsqueeze(3)\n",
        "\n",
        "    trg_len = trg.shape[1]\n",
        "\n",
        "    trg_sub_mask = torch.tril(torch.ones((trg_len, trg_len), device=device)).bool()\n",
        "\n",
        "    trg_mask = trg_pad_mask & trg_sub_mask\n",
        "\n",
        "    return src_mask, trg_mask"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "F6JuBXZC6EHf",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "enc_input_size = len(GER.vocab) # enc_vocab_size\n",
        "dec_input_size = len(ENG.vocab) # dec_vocab_size\n",
        "hidden_size = 512\n",
        "n_layers = 6\n",
        "n_head = 8\n",
        "filter_size = 2048\n",
        "dropout = 0.1\n",
        "pre_lnorm = False\n",
        "# lr = 1.5e-3\n",
        "lr = 7e-4\n",
        "betas = (0.9, 0.98)\n",
        "eps = 1e-09\n",
        "factor = 0.5\n",
        "warmup = 2000\n",
        "n_epoch = 10\n",
        "clip_value = 1"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "c3679v3Z6EHi",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model = Transformer(enc_input_size, dec_input_size, hidden_size, filter_size, n_head, dropout, n_layers, pre_lnorm, device)\n",
        "model = model.to(device)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qvcQctEp3cq5",
        "colab_type": "text"
      },
      "source": [
        "Initialize model parameters"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dgH3J1q23a98",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "for param in model.parameters():\n",
        "    if param.dim() > 1:\n",
        "        nn.init.xavier_uniform_(param)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "u0D4cN67QenB",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "criterion = nn.CrossEntropyLoss(ignore_index=GER.vocab.stoi['<pad>'])\n",
        "criterion = criterion.to(device)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Sg29yS1Udiwf",
        "colab_type": "text"
      },
      "source": [
        "### Warm up optimizer"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LijHFhMf6EHl",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class NoamOpt:\n",
        "    def __init__(self, optimizer, hidden_size, factor, warmup, step=0):\n",
        "        self.constant = hidden_size ** -0.5\n",
        "        self.factor = factor\n",
        "        self.curr_step = step\n",
        "        self._rate = 0\n",
        "        self.warmup = warmup\n",
        "        self.optimizer = optimizer\n",
        "\n",
        "    def step(self):\n",
        "        self.curr_step += 1\n",
        "        lr = self.learning_rate()\n",
        "        self._rate = lr\n",
        "        for param in self.optimizer.param_groups:\n",
        "            param['lr'] = lr\n",
        "        \n",
        "        self.optimizer.step()\n",
        "\n",
        "    def learning_rate(self, step=None):\n",
        "        if step is None:\n",
        "            step = self.curr_step\n",
        "        lr = self.factor * (self.constant * min(step ** (-0.5), step * self.warmup ** (-1.5)))\n",
        "        return lr\n",
        "\n",
        "    def zero_grad(self):\n",
        "        self.optimizer.zero_grad()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "O9cndHc47hjx",
        "colab_type": "text"
      },
      "source": [
        "This is how our learning rate changes over iterations. It reaches the peak point at warm-up steps, and then decays in inverse square root term."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VecKIuj71D_L",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 265
        },
        "outputId": "5cc8c66b-9e46-4380-bcde-ddc82fa3b127"
      },
      "source": [
        "opts = [NoamOpt(None, 512, factor, warmup)]\n",
        "total_step = n_epoch * len(train_iterator)\n",
        "plt.plot(np.arange(1, total_step), [[opt.learning_rate(i) for opt in opts] for i in range(1, total_step)])\n",
        "plt.legend([f\"{hidden_size}:{warmup}\"])\n",
        "None"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAD4CAYAAADo30HgAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0\ndHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAfnklEQVR4nO3deXxV5b3v8c/PMB8FFVBmQgiKiUSQ\nLWitem2PgNqaQ6sl2p7SHryoR2p7e+vU9h69Dq20drC3WC9VqsUhCmrJcQpa6tG2MoTBIUFKDKKJ\ngsw4QCDkd/7Yi7ATdshOspM9fd+vV16s/axnPXnWYu/9zV5r5Rdzd0RERA46KtETEBGR5KJgEBGR\nRhQMIiLSiIJBREQaUTCIiEgjXRI9gXjo16+fZ2dnJ3oaIiIpZeXKlVvdvX/T9rQIhuzsbMrKyhI9\nDRGRlGJmG6O161SSiIg0omAQEZFGFAwiItJITNcYzGwKcA+QBdzv7nc1Wd8d+CMwHtgGTHP3d4N1\nNwMzgAPAde5eeqQxzexB4DxgVzD8t9x9TWt3bP/+/VRXV7N3797WbioRevTowZAhQ+jatWuipyIi\nnaTFYDCzLGAOcAFQDawwsxJ3r4joNgPY4e65ZlYEzAammVkeUATkA4OAl8zspGCbI415vbsvbM+O\nVVdXc8wxx5CdnY2ZtWeojOXubNu2jerqakaMGJHo6YhIJ4nlVNIEoNLdq9x9H1AMFDbpUwg8FCwv\nBL5o4XfjQqDY3WvdfQNQGYwXy5jtsnfvXvr27atQaAczo2/fvvrUJZJhYgmGwcD7EY+rg7aofdy9\njvBpoL5H2LalMe80szfM7FfBaarDmNlMMyszs7ItW7ZEnbhCof10DEUyTzJefL4ZGA2cARwP3Bit\nk7vPdfeQu4f69z/s9zNEROLq3a2f8vDSjby37bNET6XDxRIMNcDQiMdDgraofcysC9CH8EXo5rZt\ndkx3/9DDaoE/ED7tlLKys7MZM2YMY8eOJRQKAbBgwQLy8/M56qijGv1i3osvvsj48eMZM2YM48eP\nZ8mSJVHHvP766xk9ejQFBQVMnTqVnTt3Nqz76U9/Sm5uLieffDKlpaUN7S+88AInn3wyubm53HXX\noXsHNmzYwMSJE8nNzWXatGns27cv3odAJOU9vbqai3/zKj/+01uc+/O/8NXf/Z35Szey/dM0fb24\n+xG/CF+grgJGAN2A14H8Jn2uBe4LlouAJ4Ll/KB/92D7KsJ3ITU7JjAw+NeAXwN3tTTH8ePHe1MV\nFRWHtSXC8OHDfcuWLY3aKioq/O233/bzzjvPV6xY0dC+atUqr6mpcXf3N9980wcNGhR1zNLSUt+/\nf7+7u99www1+ww03uLt7eXm5FxQU+N69e72qqspzcnK8rq7O6+rqPCcnx9955x2vra31goICLy8v\nd3f3yy67zB977DF3d7/qqqv83nvvPez7JcuxFOlsn+zd799/fI0Pv/EZv+x3f/dVG7f7nL+s9wt+\n+bIPv/EZH3nzsz7jweVesqbGP6utS/R0Ww0o8yjvqS3eleTudWY2CygN3tTnuXu5md0WDFoCPADM\nN7NKYHsQDgT9ngAqgDrgWnc/ABBtzOBbPmJm/YNgWANcHXPKpYhTTjklavu4ceMalvPz89mzZw+1\ntbV07974MsukSZMals8880wWLgzfwLVo0SKKioro3r07I0aMIDc3l+XLlwOQm5tLTk4OAEVFRSxa\ntIhTTjmFJUuW8OijjwIwffp0br31Vq655pr47axIiqr4YDezHlvFhq2fct0Xcrnui6PoknUU44Yd\nxzXnjWTthx+zaE0Ni9Z8wEtrP+Lo7l2YnD+AqeMGc9bIvmQdlbrX52L6PQZ3fw54rknbf0Qs7wUu\na2bbO4E7YxkzaP9CLHNqjf/7n+VUfLA7rmPmDerNLV/Ob7GfmTFp0iTMjKuuuoqZM2fGNP6TTz7J\n6aef3hAKV155JVdffXXD6aiD5s2bx7Rp0wCoqanhzDPPbFg3ZMgQamrCZ/2GDh3aqH3ZsmVs27aN\nY489li5duhzWXyRTuTsPL93I7c+u5dieXXnkyol8bmS/Rn3MjLxBvckb1JsbpoxmWdU2/rSmhuff\n3MSTq6o54ZjufKlgEF86bSDjhh6bcjdxpEURvWT217/+lcGDB/PRRx9xwQUXMHr0aM4999wjblNe\nXs6NN97I4sWLG9ruv//+w/rdeeeddOnSha9//etxn7dIJtq1Zz83PfkGz7+1ifNO6s8vvnYa/Y6O\nemNkg6yjjM/l9uNzuf24rfBUlrz9EU+vruHhpRuZ97cNDD62JxcXDOTiMQMpGNInJUIiI4Ihlp/s\nO8rgweG7cE844QSmTp3K8uXLjxgM1dXVTJ06lT/+8Y+MHDmy2X4PPvggzzzzDH/+858bnmiDBw/m\n/fcP3QVcXV3d8P2jtfft25edO3dSV1dHly5dGvUXyTSr3tvBdx5dzebde/nhRaO58vM5HNXK00E9\numZx0ZiBXDRmILv27Oelis0888YHzPvrBua+UsXQ43ty8ZhBfKlgIPmDeidtSCTj7app49NPP+Xj\njz9uWF68eDGnnnpqs/137tzJxRdfzF133cXZZ5/dbL8XXniBn/3sZ5SUlNCrV6+G9ksuuYTi4mJq\na2vZsGED69evZ8KECZxxxhmsX7+eDRs2sG/fPoqLi7nkkkswM84///yGaxQPPfQQhYVx/T1DkaRX\nX+/87uV3uOy+1zCDBVefxcxzR7Y6FJrq07MrXx0/hD98ewIrf3wBP7u0gJx+R3P/q1V86f/9lfPv\nfpmfl75NxQe7D97EkzyiXZFOta9kvSvpnXfe8YKCAi8oKPC8vDy/44473N39qaee8sGDB3u3bt38\nhBNO8EmTJrm7++233+69evXy0047reFr8+bN7u4+Y8aMhjuYRo4c6UOGDGnoc9VVVzV8zzvuuMNz\ncnL8pJNO8ueee66h/dlnn/VRo0Z5Tk5OwzwOzvGMM87wkSNH+qWXXup79+49bD+S4ViKdISPdu/1\nb9y/1Iff+Iz/+8Mrfedn+zr8e27/pNYfW7bRv3H/Us+5+VkffuMzfv7P/+Kzn1/ra97b4fX19R0+\nh4No5q4k82RLqjYIhULe9A/1rF27ttm7f6R1dCwlHf2tcivfe3wNu/fs55Yv53P5hKGdfmpn2ye1\nlJZv5tk3P2Bp1XYO1DsD+/RgUt6JTD51ABOyj6dLVsed2DGzle4eatqeEdcYREQOqjtQz69fWs+c\nlysZ2f9o5s+YwOgBvRMyl75Hd+eKicO4YuIwdn62jz+v/YgXyjdRvOJ9HnptI8f16soXTzmRyfkD\nOGdUP3p0zeqUeSkYRCRj1Ozcw3cfW03Zxh1MCw3llkvy6NUtOd4Gj+3Vja+OH8JXxw/hs311vPKP\nLZSWb2Zx+SYWrqymV7cszjupP1NOHcD5o0+gd4+OK4WfHEekg7h70l71TxXpcKpRBGBx+SauX/gG\ndQfquadoLIVjk/cOvF7dujDl1IFMOXUg+w/Us7RqG6Xlm1hcvpnn39pE1yzjzJy+/PMpJ1I4dhDH\n9uoW1++ftsHQo0cPtm3bptLb7eDB32Po0aNHoqci0ma1dQf46XNv8+Df3+XUwb357eWnk93vnxI9\nrZh1zTqKc0b155xR/bntklNZ/f5OFlds4sWKzdxSUs55J/WPezCk7cVn/QW3+NBfcJNUVrXlE77z\n2GrKP9jNv509ghsvPJnuXTrnPH1n2LjtU4b3bXvIZdzF565du+qvjolksKdXV/Pjp9+ia5ejuP+b\nIf4578RETynu2hMKR5K2wSAimenT2jr+Y1E5T66qZkL28dxz+VgG9umZ6GmlFAWDiKSN5iqiSuso\nGEQk5XkMFVEldgoGEUlpbamIKkemYBCRlBWPiqhyOAWDiKSc+nrn/79Sxd2L1zGwTw8WXH0W44Yd\nl+hppQ0Fg4iklC0f1/L9J9bw6vqtXDxmID/5yhj69NTv2cSTgkFEUkZkRdSfTB2TkIqomUDBICJJ\nL5kqomYCBYOIJLVkroiarnR0RSRppVJF1HSiYBCRpJPqFVFTnYJBRJJKuldETQUKBhFJGplQETUV\nKBhEJOFUETW5KBhEJKHWfribax9VRdRkomAQkYRwdx5e9h63P1OhiqhJRsEgIp1OFVGTm4JBRDqV\nKqImPwWDiHQKVURNHTFd4TGzKWa2zswqzeymKOu7m9njwfplZpYdse7moH2dmU1uxZi/MbNP2rZb\nIpJMtn5Sy7ceXMHsF95mSv4Anr3uHIVCEmvxE4OZZQFzgAuAamCFmZW4e0VEtxnADnfPNbMiYDYw\nzczygCIgHxgEvGRmJwXbNDummYUAPWtE0oAqoqaeWD4xTAAq3b3K3fcBxUBhkz6FwEPB8kLgixb+\nny8Eit291t03AJXBeM2OGQTRz4Eb2rdrIpJIdQfqubt0Hd94YBl9enZl0ayzuWLiMIVCCojlGsNg\n4P2Ix9XAxOb6uHudme0C+gbtS5tse7AKVnNjzgJK3P3DIz2BzGwmMBNg2LBhMeyGiHQWVURNbUn1\nP2Vmg4DLgP/RUl93nwvMBQiFQt6xMxORWKkiauqLJRhqgKERj4cEbdH6VJtZF6APsK2FbaO1jwNy\ngcrg00IvM6t099yY9kZEEkYVUdNHLMGwAhhlZiMIv3kXAVc06VMCTAdeAy4Flri7m1kJ8KiZ/ZLw\nxedRwHLAoo3p7uXAgIODmtknCgWR5KeKqOmlxWAIrhnMAkqBLGCeu5eb2W1AmbuXAA8A882sEthO\n+I2eoN8TQAVQB1zr7gcAoo0Z/90TkY6miqjpx9xT//R8KBTysrKyRE9DJKN8ti9cEXXhSlVETVVm\nttLdQ03bk+ris4ikBlVETW8KBhGJmSqiZgYFg4jERBVRM4eCQURapIqomUXBICLNUkXUzKRgEJGo\ntn5Sy/efeJ1X/rGFi8cM5CdfGUOfnl0TPS3pBAoGETlMZEXUO6eeyhUTVPwukygYRKRB3YF6fv3S\neua8XMnI/kczf8YERg/onehpSSdTMIgIoIqocoj+10VEFVGlEQWDSAZTRVSJRsEgkqE2bP2UWY+u\nUkVUOYyCQSQDqSKqHImCQSSDqCKqxELBIJIhVBFVYqVgEElzqogqraVgEEljqogqbaFgEElTqogq\nbaVgEEkz9fXO3FeruLt0HQNUEVXaQMEgkkZUEVXiQcEgkiZUEVXiRcEgkuJUEVXiTcEgksJUEVU6\ngp5BIilKFVGloygYRFKMKqJKR1MwiKQQVUSVzqBgEEkRqogqnUXBIJLkVBFVOpuCQSSJqSKqJIKC\nQSQJqSKqJJKCQSTJqCKqJFpMn0nNbIqZrTOzSjO7Kcr67mb2eLB+mZllR6y7OWhfZ2aTWxrTzB4w\ns9fN7A0zW2hmR7dvF0VSx6r3dnDRPa/yYsVmfnjRaP7wrTMUCtLpWgwGM8sC5gAXAnnA5WaW16Tb\nDGCHu+cCvwJmB9vmAUVAPjAFuNfMsloY83+5+2nuXgC8B8xq5z6KJL36eue+/3qHr933Gmaw4Oqz\nmHnuSJXJloSI5VTSBKDS3asAzKwYKAQqIvoUArcGywuB31q4elchUOzutcAGM6sMxqO5Md19d9Bm\nQE/A2757IslPFVEl2cQSDIOB9yMeVwMTm+vj7nVmtgvoG7QvbbLtwd/bb3ZMM/sDcBHh8Pnf0SZl\nZjOBmQDDhg2LYTdEko8qokoySsr73tz928AgYC0wrZk+c9095O6h/v37d+r8RNqr7kA9d5eu4xsP\nLKNPz64smnU2X584XKEgSSGWTww1wNCIx0OCtmh9qs2sC9AH2NbCtkcc090PBKeYbgD+EMM8RVLC\nBzv3cJ0qokoSi+UTwwpglJmNMLNuhC8mlzTpUwJMD5YvBZa4uwftRcFdSyOAUcDy5sa0sFxouMZw\nCfB2+3ZRJHksLt/Ehfe8ytoPd3NP0VhmX1qgUJCk0+IzMrhmMAsoBbKAee5ebma3AWXuXgI8AMwP\nLi5vJ/xGT9DvCcLXCuqAa939AEAzYx4FPGRmvQEDXgeuie8ui3Q+VUSVVGLhH+xTWygU8rKyskRP\nQyQqVUSVZGVmK9091LRdn2FFOpAqokoqUjCIdABVRJVUpmAQiTNVRJVUp2AQiRNVRJV0oWAQiQNV\nRJV0omAQaadV7+3gO4+uZvPuvfzwotFc+fkcFb+TlKZgEGmj+npn7qtV3F26jgF9erDg6rMYN+y4\nRE9LpN0UDCJtoIqoks4UDCKtpIqoku4UDCIxqjtQz69fWs+clysZ2f9o5s+YwOgBvRM9LZG4UzCI\nxEAVUSWT6Jkt0oLF5Zu4fuEb1B2o556isRSOHdzyRiIpTMEg0gxVRJVMpWAQiUIVUSWTKRhEmlBF\nVMl0CgaRgCqiioQpGEQIV0Sd9egqqlQRVUTBIJlNFVFFDqdgkIyliqgi0SkYJCOpIqpI8xQMklFU\nEVWkZQoGyRiqiCoSGwWDZARVRBWJnYJB0poqooq0noJB0pYqooq0jV4lkpZUEVWk7RQMklZUEVWk\n/RQMkjZUEVUkPhQMkhb+tLqGHz39piqiisSBgkFSmiqiisSfgkFSliqiinSMmF5FZjbFzNaZWaWZ\n3RRlfXczezxYv8zMsiPW3Ry0rzOzyS2NaWaPBO1vmdk8M9Ovpkoj7s78pRspnPM3du+t45EZE/n+\npJMVCiJx0uIrycyygDnAhUAecLmZ5TXpNgPY4e65wK+A2cG2eUARkA9MAe41s6wWxnwEGA2MAXoC\nV7ZrDyWt7Nqzn39/ZBX/509vcVZOX57/7jl8LldlskXiKZZTSROASnevAjCzYqAQqIjoUwjcGiwv\nBH5r4XoDhUCxu9cCG8ysMhiP5sZ09+cODmpmy4Ehbdw3STOqiCrSOWL57D0YeD/icXXQFrWPu9cB\nu4C+R9i2xTGDU0j/CrwQbVJmNtPMysysbMuWLTHshqSq+nrnvv96h6/d9xpmsODqs5h57kiFgkgH\nSeaLz/cCr7j7q9FWuvtcYC5AKBTyzpyYdB5VRBXpfLEEQw0wNOLxkKAtWp9qM+sC9AG2tbBts2Oa\n2S1Af+CqGOYnaUoVUUUSI5ZTSSuAUWY2wsy6Eb6YXNKkTwkwPVi+FFji7h60FwV3LY0ARgHLjzSm\nmV0JTAYud/f69u2epKK6A/XcXbqObzywjD49u7Jo1tl8feJwhYJIJ2nxE4O715nZLKAUyALmuXu5\nmd0GlLl7CfAAMD+4uLyd8Bs9Qb8nCF+orgOudfcDANHGDL7lfcBG4LXgjeApd78tbnssSS2yIurX\nQkO49ZJ8VUQV6WQW/sE+tYVCIS8rK0v0NKSdIiui/uQrY1QRVaSDmdlKdw81bdePYpJwqogqklwU\nDJJQqogqknwUDJIwqogqkpwUDNLpVBFVJLkpGKRTqSKqSPJTMEincHceXvYetz9TQZ+eXXlkxkQV\nvxNJUgoG6XC79uznpiff4Pm3NnHeSf35xddOo9/R3RM9LRFphoJBOpQqooqkHgWDdIj6emfuq1Xc\nXbqOAX16sODqsxg37LhET0tEYqBgkLhTRVSR1KZgkLhSRVSR1KdgkLioO1DPr19az5yXKxnZ/2jm\nz5jA6AG9Ez0tEWkDBYO0myqiiqQXvXqlXV6s2MwPFrxO3YF67ikaq4qoImlAwSBtooqoIulLwSCt\npoqoIulNwSCtooqoIulPwSAxUUVUkcyhYJAWqSKqSGZRMEizVBFVJDMpGCQqVUQVyVwKBjmMKqKK\nZDYFgzRQRVQRAQWDBFQRVUQOUjCIKqKKSCMKhgymiqgiEo2CIUN9sHMP3y1ezYp3VRFVRBrTO0EG\nUkVUETkSBUMGUUVUEYmFgiFDqCKqiMRKwZABVBFVRFojpkpoZjbFzNaZWaWZ3RRlfXczezxYv8zM\nsiPW3Ry0rzOzyS2NaWazgjY3MxXmaYfP9tXxgwWv873H15A3qDfPXXeOQkFEWtTiJwYzywLmABcA\n1cAKMytx94qIbjOAHe6ea2ZFwGxgmpnlAUVAPjAIeMnMTgq2aW7MvwHPAC/HYwczlSqiikhbxXIq\naQJQ6e5VAGZWDBQCkcFQCNwaLC8Efmvh35AqBIrdvRbYYGaVwXg0N6a7rw7a2rNfGcvdeWTZe9ym\niqgi0kaxBMNg4P2Ix9XAxOb6uHudme0C+gbtS5tse/DeyJbGPCIzmwnMBBg2bFhrNk1bqogqIvGQ\nshef3X0uMBcgFAp5gqeTcKqIKiLxEksw1ABDIx4PCdqi9ak2sy5AH2BbC9u2NKbEQBVRRSTeYrka\nuQIYZWYjzKwb4YvJJU36lADTg+VLgSXu7kF7UXDX0ghgFLA8xjGlBVs/qeVbD67gruffZnL+AJ69\n7hyFgoi0W4ufGIJrBrOAUiALmOfu5WZ2G1Dm7iXAA8D84OLydsJv9AT9niB8oboOuNbdD0D4ttSm\nYwbt1wE3AAOAN8zsOXe/Mq57nQZUEVVEOoqFf7BPbaFQyMvKyhI9jU7RtCLqb68Yp4qoItImZrbS\n3UNN21P24nMmUkVUEekMeldJEaqIKiKdRcGQ5FQRVUQ6m4IhiakiqogkgoIhSakiqogkioIhyXy2\nr45bFpWzYGU1Z2Qfxz1F4xh0bM9ET0tEMoiCIYmoIqqIJAMFQxJQRVQRSSYKhgRTRVQRSTYKhgRS\nRVQRSUYKhgRQRVQRSWYKhk629ZNavv/E67zyjy1cPGYgP/nKGPr07JroaYmINFAwdCJVRBWRVKBg\n6ARNK6LOnzFBFVFFJGkpGDqYKqKKSKrRO1QHUkVUEUlFCoYOoIqoIpLKFAxxtmHrp3znsVW8VbOb\nb5+dzU0XjlZFVBFJKQqGOIqsiPr7b4a4QBVRRSQFKRjiQBVRRSSdKBjaSRVRRSTdKBjaSBVRRSRd\nKRjaQBVRRSSdKRhaafV7O/jOY6vZtEsVUUUkPSkYYqSKqCKSKRQMMYisiHrRmAH89CsFqogqImlL\nwdACVUQVkUyjYGiGKqKKSKZSMEShiqgiksn0bteEKqKKSKZTMARUEVVEJCym2g1mNsXM1plZpZnd\nFGV9dzN7PFi/zMyyI9bdHLSvM7PJLY1pZiOCMSqDMbu1bxdbtmHrp3z1d3/nwb+/y7fPzubJaz6n\nUBCRjNViMJhZFjAHuBDIAy43s7wm3WYAO9w9F/gVMDvYNg8oAvKBKcC9ZpbVwpizgV8FY+0Ixu4w\nf1pdw5d+8yrVO/bw+2+GuOXL+SqTLSIZLZZPDBOASnevcvd9QDFQ2KRPIfBQsLwQ+KKF7+ksBIrd\nvdbdNwCVwXhRxwy2+UIwBsGY/9L23Wueu3PzU2/yvcfXkDeoN89dd47KZIuIEFswDAbej3hcHbRF\n7ePudcAuoO8Rtm2uvS+wMxijue8FgJnNNLMyMyvbsmVLDLtx2PaM6NeL676Qy2P/80yVyRYRCaTs\nxWd3nwvMBQiFQt6WMWaeOzKucxIRSQexfGKoAYZGPB4StEXtY2ZdgD7AtiNs21z7NuDYYIzmvpeI\niHSgWIJhBTAquFuoG+GLySVN+pQA04PlS4El7u5Be1Fw19IIYBSwvLkxg23+EoxBMOaitu+eiIi0\nVounkty9zsxmAaVAFjDP3cvN7DagzN1LgAeA+WZWCWwn/EZP0O8JoAKoA6519wMA0cYMvuWNQLGZ\n3QGsDsYWEZFOYuEf0lNbKBTysrKyRE9DRCSlmNlKdw81bdcfJxYRkUYUDCIi0oiCQUREGlEwiIhI\nI2lx8dnMtgAb27h5P2BrHKeTynQsDtGxOETH4pB0OxbD3b1/08a0CIb2MLOyaFflM5GOxSE6Fofo\nWBySKcdCp5JERKQRBYOIiDSiYAgK8QmgYxFJx+IQHYtDMuJYZPw1BhERaUyfGEREpBEFg4iINJKx\nwWBmU8xsnZlVmtlNiZ5PZzCzd83sTTNbY2ZlQdvxZvaima0P/j0uaDcz+01wfN4ws9MTO/v2M7N5\nZvaRmb0V0dbq/Tez6UH/9WY2Pdr3SnbNHItbzawmeH6sMbOLItbdHByLdWY2OaI9pV9HZjbUzP5i\nZhVmVm5m3w3aM/J50cDdM+6LcKnvd4AcoBvwOpCX6Hl1wn6/C/Rr0vYz4KZg+SZgdrB8EfA8YMCZ\nwLJEzz8O+38ucDrwVlv3HzgeqAr+PS5YPi7R+xanY3Er8IMoffOC10h3YETw2slKh9cRMBA4PVg+\nBvhHsL8Z+bw4+JWpnxgmAJXuXuXu+4BioDDBc0qUQuChYPkh4F8i2v/oYUsJ/2W9gYmYYLy4+yuE\n/15IpNbu/2TgRXff7u47gBeBKR0/+/hq5lg0pxAodvdad98AVBJ+DaX868jdP3T3VcHyx8Bawn9n\nPiOfFwdlajAMBt6PeFwdtKU7Bxab2Uozmxm0nejuHwbLm4ATg+VMOUat3f90Py6zglMk8w6ePiFD\njoWZZQPjgGVk+PMiU4MhU33e3U8HLgSuNbNzI1d6+DNxxt6/nOn7D/wOGAmMBT4EfpHY6XQeMzsa\neBL4nrvvjlyXic+LTA2GGmBoxOMhQVtac/ea4N+PgKcJnwrYfPAUUfDvR0H3TDlGrd3/tD0u7r7Z\n3Q+4ez3we8LPD0jzY2FmXQmHwiPu/lTQnNHPi0wNhhXAKDMbYWbdCP+N6pIEz6lDmdk/mdkxB5eB\nScBbhPf74B0U04FFwXIJ8M3gLowzgV0RH63TSWv3vxSYZGbHBadaJgVtKa/JNaSphJ8fED4WRWbW\n3cxGAKOA5aTB68jMjPDflV/r7r+MWJXZz4tEX/1O1Bfhuwv+Qfiuih8lej6dsL85hO8aeR0oP7jP\nQF/gz8B64CXg+KDdgDnB8XkTCCV6H+JwDB4jfIpkP+FzwDPasv/AvxG+AFsJfDvR+xXHYzE/2Nc3\nCL8BDozo/6PgWKwDLoxoT+nXEfB5wqeJ3gDWBF8XZerz4uCXSmKIiEgjmXoqSUREmqFgEBGRRhQM\nIiLSiIJBREQaUTCIiEgjCgYREWlEwSAiIo38N6qlgAaWA2TlAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "q4C1FZVnQXb4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "if pre_lnorm:\n",
        "    optimizer = optim.Adam(model.parameters(), lr=lr, betas=betas, eps=eps) # no warm up\n",
        "    lr_scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.1, patience=1, min_lr=1e-9, verbose=True)\n",
        "else:\n",
        "    # optimizer = optim.Adam(model.parameters(), lr=lr, betas=betas, eps=eps) # no warm up\n",
        "    optimizer = NoamOpt(optim.Adam(model.parameters(), lr=0, betas=betas, eps=eps), hidden_size, factor, warmup)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bz1zdLn8dhFK",
        "colab_type": "text"
      },
      "source": [
        "### Training loop and validate loop\n",
        "\n",
        "The actual target sequence is trg[:-1] = $[sos, x1, x1, ...]$, \n",
        "\n",
        "and the model output should be the prediction given the actual target token = $[y1, y2, ..., eos]$.\n",
        "\n",
        "When calculating loss, cut off the <sos> token from trg to match the output of model, which is trg[1:] = $[x1, x2, ..., eos]$"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-UX3MpVu6EHs",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def train(model, iterator, optimizer, criterion):\n",
        "    \n",
        "    model.train()\n",
        "    \n",
        "    epoch_loss = 0\n",
        "    \n",
        "    for batch in tqdm(iterator):\n",
        "        \n",
        "        src = batch.src\n",
        "        trg = batch.trg\n",
        "        \n",
        "        optimizer.zero_grad()\n",
        "        \n",
        "        src_mask, trg_mask = make_masks(src, trg[:, :-1])\n",
        "        output = model(src, src_mask, trg[:, :-1], trg_mask) # [B, T-1, output_size]\n",
        "        \n",
        "        output = output.contiguous().view(-1, output.shape[-1]) # [B*(T-1), output_size]\n",
        "        trg = trg[:, 1:].contiguous().view(-1) # [B*(T-1)]\n",
        "\n",
        "        loss = criterion(output, trg)\n",
        "        \n",
        "        loss.backward()\n",
        "        \n",
        "        torch.nn.utils.clip_grad_norm_(model.parameters(), clip_value)\n",
        "        \n",
        "        optimizer.step()\n",
        "        \n",
        "        epoch_loss += float(loss.item())\n",
        "\n",
        "        if np.isnan(epoch_loss):\n",
        "            assert False, \"gradient explode\"\n",
        "        \n",
        "    return epoch_loss / len(iterator)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LgJCXhyA6EHz",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def evaluate(model, iterator, criterion):\n",
        "    \n",
        "    model.eval()\n",
        "    \n",
        "    epoch_loss = 0\n",
        "    \n",
        "    with torch.no_grad():\n",
        "    \n",
        "        for batch in tqdm(iterator):\n",
        "\n",
        "            src = batch.src\n",
        "            trg = batch.trg\n",
        "\n",
        "            src_mask, trg_mask = make_masks(src, trg[:, :-1])\n",
        "            output = model(src, src_mask, trg[:, :-1], trg_mask) # [B, T-1, output_size]\n",
        "            \n",
        "            output = output.contiguous().view(-1, output.shape[-1]) # [B*(T-1), output_size]\n",
        "            trg = trg[:,1:].contiguous().view(-1) # [B*(T-1)]\n",
        "            \n",
        "            loss = criterion(output, trg)\n",
        "            \n",
        "            epoch_loss += float(loss.item())\n",
        "        \n",
        "    return epoch_loss / len(iterator)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RTtXtmCibkXR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def translate_sentence(model, sentence, src_field, trg_field):\n",
        "    model.eval()\n",
        "\n",
        "    if isinstance(sentence, str):\n",
        "        tokenizer = spacy.load('de')\n",
        "        tokens = [token.text.lower() for token in tokenizer(sentence)]\n",
        "    else:\n",
        "        tokens = [token.lower() for token in sentence]\n",
        "    \n",
        "    tokens = [src_field.init_token] + tokens + [src_field.eos_token]\n",
        "    src_indexes = [src_field.vocab.stoi[token] for token in tokens]\n",
        "    src_tensor = torch.LongTensor(src_indexes).unsqueeze(0).to(device)\n",
        "    \n",
        "    trg_indexes = model.inference(src_tensor)\n",
        "    trg_indexes = trg_indexes[0]\n",
        "    \n",
        "    trg_tokens = [trg_field.vocab.itos[i] for i in trg_indexes]\n",
        "    trg_tokens = trg_tokens[1:] # get rid of <sos>\n",
        "    if \"<eos>\" in trg_tokens:\n",
        "        trg_tokens = trg_tokens[:trg_tokens.index(\"<eos>\")] # get rid of <eos>\n",
        "\n",
        "    return trg_tokens"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rf5jB3Jsda5H",
        "colab_type": "text"
      },
      "source": [
        "## BLEU\n",
        "\n",
        "One of the most important benchmark for Machine Translation evaluation. I use ```corpus_bleu``` from nltk package to calculate the bleu score here. For more details, you can refer to [this article](https://machinelearningmastery.com/calculate-bleu-score-for-text-python/)."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iQ-1pprBJ69p",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from nltk.translate.bleu_score import corpus_bleu\n",
        "\n",
        "def calculate_bleu(model, data):\n",
        "\n",
        "    bleu = 0.0\n",
        "    reference = []\n",
        "    candidate = []\n",
        "    \n",
        "    for datum in tqdm(data):\n",
        "\n",
        "        src = vars(datum)['src']\n",
        "        trg = vars(datum)['trg']\n",
        "\n",
        "        # src_mask, trg_mask = make_masks(src, trg[:, :-1])\n",
        "\n",
        "        pred = translate_sentence(model, src, GER, ENG)\n",
        "\n",
        "        reference.append([trg])\n",
        "        candidate.append(pred)\n",
        "\n",
        "    bleu = corpus_bleu(reference, candidate)\n",
        "    bleu *= 100\n",
        "\n",
        "    return bleu"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mVdisDih6EH2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def calculate_time(start_time, end_time):\n",
        "    second = end_time - start_time\n",
        "    hour = math.floor(second / 3600)\n",
        "    minute = second - hour * 3600\n",
        "    minute = math.floor(minute / 60)\n",
        "    second -= (minute * 60 + hour * 3600)\n",
        "    \n",
        "    return hour, minute, second"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0qeoSdnz6EH5",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "6f39a89c-0269-4282-a9a5-9290f01501bf"
      },
      "source": [
        "best_valid_loss = float('inf')\n",
        "epoch_record = []\n",
        "bleu_score_record = []\n",
        "train_loss_record = []\n",
        "valid_loss_record = []\n",
        "\n",
        "start_time = time.time()\n",
        "\n",
        "for epoch in range(n_epoch):\n",
        "    # if pre_lnorm:\n",
        "    #     if epoch+1 == 6:\n",
        "    #         for param in optimizer.param_groups:\n",
        "    #             param['lr'] *= 0.1\n",
        "    \n",
        "    train_loss = train(model, train_iterator, optimizer, criterion)\n",
        "    valid_loss = evaluate(model, valid_iterator, criterion)\n",
        "    bleu_value = calculate_bleu(model, test_data)\n",
        "    # bleu_value = 0\n",
        "    \n",
        "    if pre_lnorm:\n",
        "        lr_scheduler.step(valid_loss)\n",
        "\n",
        "    end_time = time.time()\n",
        "    \n",
        "    hour, minute, second = calculate_time(start_time, end_time)\n",
        "    \n",
        "    if valid_loss < best_valid_loss:\n",
        "        best_valid_loss = valid_loss\n",
        "        torch.save(model.state_dict(), 'transformer.pt')\n",
        "\n",
        "    epoch_record.append(epoch+1)\n",
        "    train_loss_record.append(train_loss)\n",
        "    valid_loss_record.append(valid_loss)\n",
        "    bleu_score_record.append(bleu_value)\n",
        "\n",
        "    print('-----------------------------------------------------------------------------------------------')\n",
        "    print(f'Epoch: {epoch+1:02} | Train Loss: {train_loss:.3f} | Validation Loss: {valid_loss:.3f} | Bleu Score: {bleu_value:.2f} | Since: {hour}h {minute}m {second:.0f}s')\n",
        "    print(f'lr: {optimizer._rate:2f} | #step: {optimizer.curr_step}')\n",
        "    print('-----------------------------------------------------------------------------------------------')\n",
        "    time.sleep(0.5)"
      ],
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 227/227 [01:01<00:00,  4.14it/s]\n",
            "100%|██████████| 8/8 [00:00<00:00, 13.46it/s]\n",
            "100%|██████████| 1000/1000 [07:40<00:00,  2.16it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "-----------------------------------------------------------------------------------------------\n",
            "Epoch: 01 | Train Loss: 6.469 | Validation Loss: 4.631 | Bleu Score: 2.23 | Since: 0h 8m 43s\n",
            "lr: 0.000056 | #step: 227\n",
            "-----------------------------------------------------------------------------------------------\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 227/227 [01:01<00:00,  3.85it/s]\n",
            "100%|██████████| 8/8 [00:00<00:00, 12.69it/s]\n",
            "100%|██████████| 1000/1000 [07:47<00:00,  2.16it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "-----------------------------------------------------------------------------------------------\n",
            "Epoch: 02 | Train Loss: 4.203 | Validation Loss: 3.593 | Bleu Score: 9.18 | Since: 0h 17m 34s\n",
            "lr: 0.000112 | #step: 454\n",
            "-----------------------------------------------------------------------------------------------\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 227/227 [01:01<00:00,  3.78it/s]\n",
            "100%|██████████| 8/8 [00:00<00:00, 12.42it/s]\n",
            "100%|██████████| 1000/1000 [07:42<00:00,  2.22it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "-----------------------------------------------------------------------------------------------\n",
            "Epoch: 03 | Train Loss: 3.458 | Validation Loss: 3.088 | Bleu Score: 12.56 | Since: 0h 26m 20s\n",
            "lr: 0.000168 | #step: 681\n",
            "-----------------------------------------------------------------------------------------------\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 227/227 [01:01<00:00,  4.11it/s]\n",
            "100%|██████████| 8/8 [00:00<00:00, 11.99it/s]\n",
            "100%|██████████| 1000/1000 [07:32<00:00,  2.23it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "-----------------------------------------------------------------------------------------------\n",
            "Epoch: 04 | Train Loss: 3.010 | Validation Loss: 2.780 | Bleu Score: 15.23 | Since: 0h 34m 56s\n",
            "lr: 0.000224 | #step: 908\n",
            "-----------------------------------------------------------------------------------------------\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 227/227 [01:01<00:00,  3.67it/s]\n",
            "100%|██████████| 8/8 [00:00<00:00, 12.93it/s]\n",
            "100%|██████████| 1000/1000 [07:32<00:00,  2.21it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "-----------------------------------------------------------------------------------------------\n",
            "Epoch: 05 | Train Loss: 2.693 | Validation Loss: 2.591 | Bleu Score: 16.48 | Since: 0h 43m 32s\n",
            "lr: 0.000280 | #step: 1135\n",
            "-----------------------------------------------------------------------------------------------\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 227/227 [01:01<00:00,  3.80it/s]\n",
            "100%|██████████| 8/8 [00:00<00:00, 12.98it/s]\n",
            "100%|██████████| 1000/1000 [07:32<00:00,  2.23it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "-----------------------------------------------------------------------------------------------\n",
            "Epoch: 06 | Train Loss: 2.450 | Validation Loss: 2.442 | Bleu Score: 17.64 | Since: 0h 52m 8s\n",
            "lr: 0.000336 | #step: 1362\n",
            "-----------------------------------------------------------------------------------------------\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 227/227 [01:01<00:00,  3.39it/s]\n",
            "100%|██████████| 8/8 [00:00<00:00, 12.18it/s]\n",
            "100%|██████████| 1000/1000 [07:32<00:00,  2.21it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "-----------------------------------------------------------------------------------------------\n",
            "Epoch: 07 | Train Loss: 2.242 | Validation Loss: 2.343 | Bleu Score: 19.34 | Since: 1h 0m 43s\n",
            "lr: 0.000393 | #step: 1589\n",
            "-----------------------------------------------------------------------------------------------\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 227/227 [01:01<00:00,  3.61it/s]\n",
            "100%|██████████| 8/8 [00:00<00:00, 12.29it/s]\n",
            "100%|██████████| 1000/1000 [07:31<00:00,  2.24it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "-----------------------------------------------------------------------------------------------\n",
            "Epoch: 08 | Train Loss: 2.069 | Validation Loss: 2.259 | Bleu Score: 19.11 | Since: 1h 9m 18s\n",
            "lr: 0.000449 | #step: 1816\n",
            "-----------------------------------------------------------------------------------------------\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 227/227 [01:01<00:00,  4.02it/s]\n",
            "100%|██████████| 8/8 [00:00<00:00, 11.90it/s]\n",
            "100%|██████████| 1000/1000 [07:31<00:00,  2.21it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "-----------------------------------------------------------------------------------------------\n",
            "Epoch: 09 | Train Loss: 1.925 | Validation Loss: 2.224 | Bleu Score: 20.30 | Since: 1h 17m 53s\n",
            "lr: 0.000489 | #step: 2043\n",
            "-----------------------------------------------------------------------------------------------\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 227/227 [01:00<00:00,  3.83it/s]\n",
            "100%|██████████| 8/8 [00:00<00:00, 12.82it/s]\n",
            "100%|██████████| 1000/1000 [07:33<00:00,  2.19it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "-----------------------------------------------------------------------------------------------\n",
            "Epoch: 10 | Train Loss: 1.767 | Validation Loss: 2.152 | Bleu Score: 20.63 | Since: 1h 26m 28s\n",
            "lr: 0.000464 | #step: 2270\n",
            "-----------------------------------------------------------------------------------------------\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nxa3anl9Ieoh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "file_path = '/content/drive/My Drive/transformer_checkpoint/'\n",
        "if pre_lnorm:\n",
        "    torch.save(epoch_record, file_path+'epoch_record_pre.pkl')\n",
        "    torch.save(bleu_score_record, file_path+'bleu_score_record_pre.pkl')\n",
        "    torch.save(train_loss_record, file_path+'train_loss_record_pre.pkl')\n",
        "    torch.save(valid_loss_record, file_path+'valid_loss_record_pre.pkl')\n",
        "else:\n",
        "    torch.save(epoch_record, file_path+'epoch_record_post.pkl')\n",
        "    torch.save(bleu_score_record, file_path+'bleu_score_record_post.pkl')\n",
        "    torch.save(train_loss_record, file_path+'train_loss_record_post.pkl')\n",
        "    torch.save(valid_loss_record, file_path+'valid_loss_record_post.pkl')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OPkMxv3aW34K",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "file_path = '/content/drive/My Drive/transformer_checkpoint/'\n",
        "\n",
        "epoch_record_pre = torch.load(file_path+'epoch_record_pre.pkl')\n",
        "bleu_score_record_pre = torch.load(file_path+'bleu_score_record_pre.pkl')\n",
        "train_loss_record_pre = torch.load(file_path+'train_loss_record_pre.pkl')\n",
        "valid_loss_record_pre = torch.load(file_path+'valid_loss_record_pre.pkl')\n",
        "\n",
        "epoch_record_post = torch.load(file_path+'epoch_record_post.pkl')\n",
        "bleu_score_record_post = torch.load(file_path+'bleu_score_record_post.pkl')\n",
        "train_loss_record_post = torch.load(file_path+'train_loss_record_post.pkl')\n",
        "valid_loss_record_post = torch.load(file_path+'valid_loss_record_post.pkl')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "D0mrH_k4FCKz",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 284
        },
        "outputId": "d4e7c480-1774-44e4-cb13-a48217368044"
      },
      "source": [
        "plt.plot(epoch_record, valid_loss_record_post, 'x--', label='Post-LN (Adam w/ warm-up)')\n",
        "plt.plot(epoch_record, valid_loss_record_pre, '.-', color=\"red\", label='Pre-LN (Adam w/o warm-up)')\n",
        "# plt.title('')\n",
        "plt.xlabel('Epochs', fontsize=16)\n",
        "plt.ylabel('Validation Loss', fontsize=16)\n",
        "plt.legend()\n",
        "plt.grid()\n",
        "plt.show()"
      ],
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAELCAYAAAAybErdAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0\ndHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOzdd3gU1frA8e/JphdCICGEBEihKUkg\nEAkJhCoIFhAVRVRE7OVarwVFFOwXrqIXfyoqFqRY8XotICChBpBm6CUQIIBAaCGB9PP7YzZLQgob\nUibl/TzPPrs7MzvzZgj7Zuac8x6ltUYIIYQoysHsAIQQQtQ+khyEEEKUIMlBCCFECZIchBBClCDJ\nQQghRAmOZgdQFXx9fXVwcLDZYVRKZmYmHh4eZodRa8j5KE7Ox3lyLoqrzPlYt25dmtbar7R19SI5\nBAcHs3btWrPDqJSEhAT69Oljdhi1hpyP4uR8nCfnorjKnA+l1L6y1sltJSGEECVIchBCCFGCJAch\nhBAl1Is2ByGqW25uLqmpqWRlZZlyfG9vb7Zt22bKsWsbORfF2XM+XF1dCQoKwsnJye79SnIQwg6p\nqal4eXkRHByMUqrGj3/mzBm8vLxq/Li1kZyL4i52PrTWHD9+nNTUVEJCQuzeb4O8rfThkmRWJqcV\nW7YyOY0PlySbFJGo7bKysmjatKkpiUGIylBK0bRp0wpf9TbI5BAZ5M0jszbYEsTK5DQembWByCBv\nkyMTtZkkBlFXXcrvboO8rRQX5svUkVE8MmsDt8W0Yuaq/Uy9LYq4MF+zQxNCiFqhQV45gJEghkUF\n8p8/dhMd7COJQdR6FouFzp07Ex4ezvDhwzl79myF9zFlypQyP5eQkMC1115bYnmfPn2Ijo62vV+7\ndm2Zg64OHz5cYh+PP/44gYGBFBQUlBlXcHAwaWlpZa6vTg888AArVqww5dhVZdOmTYwePbpK99lg\nk8PK5DTmbjhIEw8nFm07ysrd5vxiivqnutq03Nzc2LhxI5s3b8bZ2ZkPP/ywwvsoLzmU5+jRo/z2\n228X3e7tt9/m3nvvtb0vKChg7ty5tGzZkiVLllT4uDVh1apVdO/evcr3m5eXV+X7LEtERASpqans\n37+/yvbZIJNDYRvD1JFRPH/15eRrzf1frSvxH1qIS1ETbVrx8fHs3r0bML6Qw8PDCQ8PZ8qUKYBR\nb+eaa66hU6dOhIeH8/XXX/Pee+9x6NAh+vbtS9++fSt0vKeffprXXnvtott9//33DBo0yPY+ISGB\njh078uCDDzJ79mzb8uPHjzNw4EA6duzIPffcQ9EZKa+//nq6du1Kx44dmTZtmm25p6cnTz/9NN26\ndePKK69kzZo19OnTh9DQUH766acSsTz88MO25cOGDWPMmDEATJ8+nRdeeAGAbdu20a5dOywWi+1z\n+fn5hISEoLXm1KlTWCwWli5dCkCvXr3YtWsXa9asITY2lqioKOLi4tixYwcAn3/+OUOGDKFfv370\n79+fhIQEevfuzdChQwkNDeW5555j5syZdOvWjYiICJKTS/+DwdPT0/b6u+++s10VjB49mgceeIDo\n6GjatWvHzz//bNvuuuuuY86cOeX981RIg2xzSEo9zdSRRhtDTl4Bk+fvoKmnM0mpp+X2krDLLR8l\nllh2bWQAd8QGE9XSh2ZeLoz6dA3+jVw4kp5Nm2aeHDx5DoATmTk8+NW6Yp/9+v5Yu4+dl5fHb7/9\nxqBBg1i3bh2fffYZq1evRmtNTEwMvXv3Zs+ePbRo0YJffvkFgNOnT+Pt7c3bb7/N4sWL8fWt2O95\nbGwsc+fOZfHixWV2m9y7dy8+Pj64uLjYls2ePZtbb72VoUOH8vzzz5Obm4uTkxMTJkygZ8+ejB8/\nnl9++YVPP/3U9pnp06fTpEkTzp07xxVXXMGNN95I06ZNyczMpF+/fowfP55Ro0Yxbtw4FixYwNat\nW7nzzjsZMmRIsXji4+NZtmwZQ4YM4eDBgxw+fBiAZcuWMWLECADbeSzKYrHQvn17tm7dyt69e+nS\npQvLli0jJiaGAwcO0LZtW9LT01m2bBmOjo4sXLiQ559/nu+//x6A9evXk5SURJMmTUhISOCvv/5i\n27ZtNGnShNDQUO655x7WrFnDu+++y3/+8x9bQrdXSkoKa9asITk5mb59+7Jhwwa8vLyIjo7mzTff\n5JlnnqnQ/srSIK8cHugdZksCzo4O3N0zhC2H0okLa2pyZKK+8HZzwr+RCwdPZeHfyAVvN/sHH5Xl\n3LlzdO7cmejoaFq1asXdd9/N8uXLGTZsGB4eHnh6enLDDTewbNkyIiIiWLBgAc8++yzLli3D27vy\nVy3jxo3j1VdfLXP94cOH8fM7X+AzJyeHX3/9leuvv55GjRoRExPD/PnzAVi6dCm33347ANdccw0+\nPj62z7333nt06tSJ7t27c+DAAXbt2gWAs7Oz7Ys8IiKC3r174+TkREREBCkpKSXiKUwOW7du5fLL\nL8ff35/Dhw+TmJhIXFwcAPPnzy+RHAo/u3TpUpYuXcrYsWNZvnw5f/75J1dccQVgJNvhw4cTHh7O\nE088wZYtW2yfHTBgAE2aNLG9v+KKKwgICMDFxYWwsDAGDhxo+xlKi/tibr75ZhwcHGjbti2hoaHs\n3LkTgGbNmnHo0KEK768sDfLK4UIjurXEy9WRdv4ysEbYp7y/9N2cLTx2ZVsembWBR/u14avV+3ns\nyra2P0iaeDhX6ErBtl9rm4M92rVrx/r16/n1118ZN24c/fv3Z/z48cW2mTt3LhMmTADgk08+ueg+\n+/Xrx7hx41i1alWZ8RXtSz9//nxOnTpFREQEAGfPnsXNza3URu9CCQkJLFy4kMTERNzd3enTp49t\nn05OTrYumQ4ODrYrFAcHh1Lv7wcGBnLq1CnmzZtHr169OHHiBN988w2enp54eXlx9uxZTp06RYsW\nLUp8tlevXnzwwQccOnSIiRMnMmnSJBISEoiPjwfgxRdfpG/fvsydO5eUlJRiDfQXls8ueiVVWtz5\n+fl07doVgCFDhjBx4sRiXU8vHJ9wYbfUwvdZWVm4ubmVdWorzJQrB6WURSm1QSn1cynrRiuljiml\nNlof91R3PF6uTozo1gpXJ8vFNxbiIoq2aT05sL2t23R1tGnFx8fz448/cvbsWTIzM5k7dy7x8fEc\nOnQId3d3br/9dp5++mnWr18PgJeXF2fOnAGM+/AbN25k48aNxXojlWfcuHH861//KnVdu3btiv0l\nPHv2bD755BNSUlJISUlh7969LFiwgLNnz9KrVy9mzZoFGLd2Tp48CRh/kfv4+ODu7s727dvLTET2\n6t69O1OmTKFXr17Ex8czefJk2xf84sWLy2x76datGytXrsTBwQFXV1c6d+7MRx99RK9evWxxBgYG\nAkY7Q2VYLBbbv8PEiRMB8Pf3Z9u2bbYG/aK+/fZbCgoKSE5OZs+ePbRt2xaAnTt3Eh4eXqlYijLr\nttJjQHnFQL7WWne2Pi7+J00V+WrVPv6zaFdNHU7UU0XbtOD8uJqk1NNVfqwuXbowevRounXrRkxM\nDPfccw9RUVFs2rSJbt260blzZyZMmMC4ceMAuO+++xg0aFCZX4qLFi0iKCjI9khMLN62cvXVVxe7\ndVSUh4cHYWFh7N69m7NnzzJv3jyuueaaYut79uzJ//73P1566SWWLl1Kx44d+eGHH2jVqhUAgwYN\nIi8vj8suu4znnnuu0r2I4uPjycvLo02bNnTp0oUTJ07YkkNp7Q2FXFxcaNmype348fHxnDlzxnYV\n9MwzzzB27FiioqKqpVfSm2++ybXXXktcXBwBAQHF1rVq1Ypu3boxePBgPvzwQ1xdXQEj2RU935Wm\nta7RBxAELAL6AT+Xsn40MLUi++zatauuCs98+5duP+5XnXYmq0r2VxGLFy+u8WPWZrXtfGzdutXU\n46enp5t6fHv98MMP+oUXXqjWY1TVuYiKitI5OTlVsq+acuedd+pvv/222LL09HSdlZWlY2JidG5u\nbpmfLe13GFiry/heNePKYQrwDFD2iBi4USmVpJT6TinVsobi4t5eIWTlFvBlYpmTIwkhyjFs2DDq\nypS969evr1CV0tps//79vPnmmzg6Vl0zstJF+hdXN6XUtcDVWuuHlFJ9gH9qra+9YJumQIbWOlsp\ndT9wi9a6Xyn7ug+4D8Df379rVfXvfXd9FrtO5vPv3u64ONZcLZ2MjIxifZsbutp2Pry9vWnTpo1p\nx8/Pzy/WF78hk3NRnL3nY/fu3Zw+XfzWZt++fddprUttcKrp3ko9gCFKqasBV6CRUuorrfXthRto\nrY8X2f4ToNTWL631NGAaQHR0tK6qOWU9g09w04eJ/O0ewp1xwVWyT3vIvLjF1bbzsW3bNlPLREuZ\n6vPkXBRn7/lwdXUlKirK7v3W6G0lrfVYrXWQ1joYGAH8UTQxACilira+DKH8husqFx3chFGxrQnz\nqz1/tQohRE2rFeMclFITMRpGfgIeVUoNAfKAExgN1DVq4tCq6w4mhBB1kWnJQWudACRYX48vsnws\nMNacqM47diabn5MOMTrOnJm/hBDCTA2yfIY9Fm07woT/bWVl8vGLbyxEDaiKkt2FUlJSSh0wNXr0\naAIDA8nOzgYgLS2tzN5H586do3fv3uTn59uWTZkyBVdX1xINn0X16dOHtWvXXnLslfHmm28yc+ZM\nU45d3Y4dO1bmuI1LIcmhDNdHBeLn5SJTh4pa42Ilu7XW5c6ZYC+LxcL06dMvut306dO54YYbivWU\nmT17NldccQU//PBDpeOoDvPnz7fVNqoJNVm228/Pj4CAgCqbm0KSQxlcnSzc1SOYZbvS2HKo6ke2\nigYgMRHeeMN4rmKFJbtTUlJo3749o0aNIjw8nAMHDvD7778TGxtLly5dGD58OBkZGRXa9+OPP847\n77xz0S+2mTNnMnToUNv75ORkMjIyePXVV4uV5z537hwjRozgsssuY9iwYZw7d8627sEHHyQ6OpqO\nHTvy0ksv2ZYHBwczduxYW6HB9evXc9VVVxEWFlasgmuhSZMm8d577wHwxBNP0K+f0fv9jz/+4Lbb\nbgMgPT2dnJwc/Pz8SElJoV+/fkRGRtK/f/9S50GIiIjg1KlTaK1p2rQpX375JQCjRo1iwYIFpKSk\nEB8fT5cuXejSpQsrV64EsNVgGjJkCJdffjkpKSl06NCB0aNH065dO2677TYWLlxIjx49aNu2LWvW\nrCn1/BadAKnoBEsvv/wyd9xxB7GxsbRt27ZY+Y7rr7++yq6MJDmU47aY1ng4W5i2dI/ZoYja5PHH\noU+f8h9RUdCzJzz/vPEcFVX+9o8/bvfhC0t2F5Zy2LVrFw899BBbtmzBw8ODV199lYULF7J+/Xqi\no6N5++23K/TjtWrVip49ezJjxowyt8nJyWHPnj3FbjnNmTOHESNGEB8fz44dOzhy5AgAH3zwAe7u\n7mzbto0JEyawbt35cuWvvfYaa9euJSkpiSVLlpCUlFQsjo0bNxIfH8/o0aP57rvvWLVqFa+//nqJ\neAorsILxRZqRkUFubi7Lli2z1UNauHAh/fv3B+Af//gHd955J0lJSdx22208+uijJfbZo0cPVqxY\nwZYtWwgNDbXtv7Cqa7NmzViwYAHr16/n66+/LraP9evX8+6779oqpu7evZunnnqK7du3s337dmbN\nmsXy5cuZPHlyqT/PxSQlJfHHH3+QmJjIW2+9ZavGGh0dbYuzsiQ5lMPbzYnbu7cmv0BTUFBzgwVF\nPXD6NBTe4ikoMN5XUmkluwFat25tqwG0atUqtm7dSo8ePejcuTNffPEF+/ZVfMT/2LFjmTRpUpm3\nqdLS0mjcuHGxZbNnz2bEiBE4ODhw44038u233wLFy3NHRkYSGRlp+8w333xDly5diIqKYsuWLWzd\nutW2rnB+hoiICGJiYvDy8sLPzw8XFxdOnTpV7Nhdu3Zl3bp1pKen4+LiQmxsLGvXrmXZsmW2Wkrz\n5s1j8ODBgPEFP3LkSADuuOMOli9fXuJnLFq2+8EHH2TTpk0cPHgQHx8fPDw8yM3N5d577yUiIoLh\nw4cXi71bt26EhITY3oeEhBAREYGDgwMdO3akf//+KKUuuWz30KFDcXNzw9fXl/j4eNvVR1WW7a4V\nXVlrs+cGd5DeSqI4eyZnSUyE/v0hJwecnWHmTIiteJnuosoq2V20RLTWmgEDBhS7rQOwevVq7r//\nfgAmTpxY7Au6NG3btqVz58588803ZcZStJT0pk2b2LVrFwMGDACMK4uQkBAeeeSRMo+xd+9eJk+e\nzJ9//omPjw+jR48uts+ipa0vLHt94S0vJycnQkJC+Pzzz4mLiyMyMpLFixeze/duLrvsMgDWrFnD\nBx98UO7PXVSvXr14//332b9/P6+99hpz587lu+++syWbd955B39/f/766y8KCgpsBfCg4mW7Aa66\n6iqOHDlCdHQ0n3zyCY6OjrbkbEbZbrlyuIjCk558LIPTZ3NNjkbUGbGxsGgRvPKK8VzJxGCv7t27\ns2LFCtsUopmZmezcuZOYmBhbWegLZ0wrywsvvMDkyZNLXefj40N+fr7tS2v27Nm8/PLLtvLchw4d\n4tChQ+zbt69Yee7Nmzfbbh2lp6fj4eGBt7c3R44csWuO6vIUluQuLM/94YcfEhUVhVKKLVu20KFD\nB1vjeVxcnG1KzZkzZ9q+8Itq2bIlaWlp7Nq1i9DQUHr27GnbPxhluwMCAnBwcGDGjBnFem1divnz\n57Nx40bb3BrBwcG2W3CFs8wV+u9//0tWVhbHjx9n+fLltkmIqrJstyQHOxw8dY4Bby/hy8QUs0MR\ndUlsLIwdW2OJAYweK59//jm33norkZGRxMbGsn379lK33bFjR7Hy3IW3gQp17NiRLl26lHmsgQMH\n2m7HzJkzh2HDhhVbP2zYMObMmcODDz5IRkYGl112GePHj7dNbNOpUyeioqLo0KEDI0eOpEePHpX5\n0YmPj+fw4cPExsbi7++Pq6trmeW5//Of//DZZ58RGRnJjBkzePfdd0vdZ0xMDO3atbPt/+DBg/Ts\n2ROAhx56iC+++IJOnTqxffv2ElcLlfXSSy/x2GOPER0dXaJ2UmRkJH379qV79+4888wztgmLqrRs\nd1nlWuvSo6pKdpdn9PTVusvE3/W5nLxq2X9tK1Ftttp2PqRkd0nr1q3Tt99+e40f91LOxZVXXqkP\nHTpUDdHUvJdeeklPmjTJ9r7o+YiPj9cnTpwo9XN1oWR3nXR/7zCOZ+bw3bpUs0MRolbo0qULffv2\nrfTtlJqwYMGCEpPm1DfHjh3jySefLDYfd2VIg7SdYkKa0CnIm4+X7eHWbq2wOEgjtRBjxowxO4QG\n5+WXXy51uZ+fH9dff32VHUeuHOyklOL+3mH8fTqLbYfTzQ5HmEDX4NwnQlSlS/ndleRQAVd1bM6K\n5/oRHuhtdiiihrm6unL8+HFJEKLO0Vpz/PjxYl1t7SG3lSrA4qDw9XRBa83ZnHw8XOT0NRRBQUGk\npqZy7NgxU46flZVV4f/c9ZWci+LsOR+urq4EBQVVaL/y7XYJ7p+xjtz8Aj67q5vZoYgaUjjIyiwJ\nCQkVmsWrPpNzUVx1nQ+5rXQJIgK9WbzjGNv/lrYHIUT9JMnhEtzevTVuTlKQTwhRf0lyuAQ+Hs7c\nckVLftp4iMOnz138A0IIUcdIcrhEd/cMQQMzEite8VIIIWo7aZC+RC2buDN99BV0C25idihCCFHl\nJDlUQu92fmaHIIQQ1UJuK1XS4u1HufnDRLJya399GSGEsJckh0pydnRgTcoJftxw0OxQhBCiykhy\nqKS4sKZ0bNGIacv2yFSiQoh6Q5JDJRUW5NtzLJOF246YHY4QQlSJS04OSqnGSqlOSimnqgyoLro6\nvDlBPm58JIPihBD1hF29lZRSYwFPrfUL1vc9gV8AT+CAUupKrfXu6guzdnO0OPDC1ZehMSogXjj5\ntxBC1DX2XjmMAoqO9voXsBm4CTgOTKzIQZVSFqXUBqXUz6Wsc1FKfa2U2q2UWq2UCq7Ivs0yOCKA\nqyMCJDEIIeoFe5NDELALQCnlC8QA47TWc4E3gN4VPO5jwLYy1t0NnNRatwHeAd6q4L5Nk56Vy7sL\nd7H7aIbZoQghRKXYmxzygcK2hV5AFrDC+v4oYPcwYaVUEHAN8EkZmwwFvrC+/g7or+rIn+O5eQX8\nX8JuPlkmbQ9CiLrN3hHSW4FblVLLgLuApVrrHOu6lkBFZkCZAjwDeJWxPhA4AKC1zlNKnQaaAmlF\nN1JK3QfcB+Dv709CQkIFQqg+PVo48N3aA3T3SKOxq/3t/RkZGbXmZ6gN5HwUJ+fjPDkXxVXX+bA3\nObwC/IjR9pAHDCqybhCw3p6dKKWuBY5qrdcppfpUIM4StNbTgGkA0dHRuk+fSu2uyoREZNJ3cgI7\naMGzfTrY/bmEhARqy89QG8j5KE7Ox3lyLoqrrvNh15+2WuvfgI7ASCBCa724yOpEjAZqe/QAhiil\nUoA5QD+l1FcXbHMQ42oEpZQj4I3R6F0ntG7qweCIAL5atY8zWblmhyOEEJfE7sJ71q6qJbqraq3/\nrwL7GAuMBbBeOfxTa337BZv9BNyJkXRuAv7QdWxW9/t7hZKVk8/pc7l4uTb4YSBCiDrIrisHpdS1\nSqk7i7xvqZRappQ6qZSao5TyqEwQSqmJSqkh1refAk2VUruBJ4HnKrNvM0QGNebT0VcQ5ONudihC\nCHFJ7G0xHQ80L/L+HSAU+BLoD7xU0QNrrRO01tdaX4/XWv9kfZ2ltR6utW6jte6mta6zXX/2Hc9k\n/f6TZochhBAVZm9yaAP8BaCUcgOuBp7UWj8GPA/cUD3h1W0PfrWe575Poo7dFRNCCLuTgytQOFly\nLOAMzLe+3w60qOK46oV74kPYeSSDhB0V6ekrhBDmszc5pABx1tdDgHVa61PW937AmSqOq164rlML\nWni78uGSZLNDEUKICrE3OXwCTFBKrQIeBqYXWReLMUhOXMDJ4sCYniGs3nuCjQdOXfwDQghRS9g7\nzuFt4F5gA3C/1vqjIqt9gM+rPrT6YUS3Vvh6urD54GmzQxFCCLtVZJzDF5yveVR0+T1VGlE94+ni\nyPJn++LqZDE7FCGEsJvdyQFAKTUIowJrE+AEsFhr/Xt1BFafFCaGg6fOEdjYzeRohBDi4uwdBOeh\nlPoD+BV4GqPr6tPAb0qpRUopGe11ER8tSabf5ASOnck2OxQhhLgoexuk3wC6YVRkddda+wFuwBjr\n8terJ7z648rL/cnJL+DLxBSzQxFCiIuyNznciDG5zxeFpbq11rnWdojxGDWQRDnC/DwZeLk/Xybu\nIzM7z+xwhBCiXPYmh6YY04KWZjPgWzXh1G/39w7j9Llcvv7zgNmhCCFEuexNDvswZm8rzSCMQXLi\nIrq08uGKYB++XZcqJTWEELWavb2VpgGTrA3PM4HDGIX4RgD3Y8zsJuww6aZO+Hq5UEdmPhVCNFB2\nJQet9b+VUv7AY0DRcQ15wL+tg+SEHYJ9jermBQUapZAkIYSoleye5Fhr/QzG/M7XY/RSuh4I1Fo/\nW02x1VspaZkMnLKUZbvSLr6xEEKYwO7kAKC1TtNa/8/aa+l/Wus0pVR3pdQP1RVgfRTQ2JUzWbl8\ntFQK8gkhaqcKJYcyBAJDq2A/DYaLo4UxPUJYsfu41FwSQtRKVZEcxCW4NaYVni6OfLS0zk50J4So\nxyQ5mKSRqxO3xbTil6RDHDhx1uxwhBCimAoV3hNVa0zPEC5v0YgAb1ek9UEIUZvIlYOJ5m44iJ+X\nC46W8/8MK5PTZOY4IYTpyrxyUEoVADKMtxpFBnnzyKwNDLzcH8uZXJyT03hk1gamjowyOzQhRANX\n3m2l15HkUK3iwnyZOjKKO6evAa35ed96Pri9C3FhUqpKCGGuMpOD1npcTQbSUMWF+TLiipbMWLWf\n/IICWjWRqTGEEOaTNgeTrUxO45dNf9M7yEJmdj43frCSo+lZZoclhGjgJDmYaGWRNoa7wl15aUhH\njqZnM3uNlPQWQpirRpODUspVKbVGKfWXUmqLUmpCKduMVkodU0pttD7uKW1f9UFS6mmmjoyytTGM\njgtm2h1dcXEy/lmkrLcQwiw1Pc4hG+intc5QSjkBy5VSv2mtV12w3dda60dqOLYa90DvsBLLBnRs\nzgBg4dYjfJGYwrQ7onFzttR4bEKIhq1Grxy0IcP61sn6kD+PS3EuN5/lu9N44Kt1ZOflmx2OEKKB\nUTV960IpZQHWAW2A9y8s+a2UGg28ARwDdgJPaK1L3IRXSt0H3Afg7+/fdc6cOdUcefXKyMjA09Oz\n2LIlqbl8tjmHrv4WHurkgsWh4cz9UNr5aMjkfJwn56K4ypyPvn37rtNaR5e2zu7koIxZaboCrQDX\nC9drrWdVJCilVGNgLvAPrfXmIsubAhla62yl1P3ALVrrfuXtKzo6Wq9du7Yih691EhIS6NOnT4nl\nny7fyys/b+WGqEAmD++EQwNJEGWdj4ZKzsd5ci6Kq8z5UEqVmRzsanNQSnXA+CJvB5T27aSBCiUH\nrfUppdRijDmoNxdZfrzIZp8A/6rIfuubu3uGkJmdx/GMbLNDEUI0IPY2SP8f4AaMBDZhNCxXmFLK\nD8i1JgY3YADw1gXbBGitD1vfDgG2Xcqx6pN/9GsDGFOKnjqbQ2N3Z5MjEkLUd/Ymh2hgjNb6u0oe\nLwD4wtru4AB8o7X+WSk1EVirtf4JeFQpNQRjfuoTwOhKHrPOK5xn+u/TWVz//gruiG3Nw33bmByV\nEKI+szc5HAfOVfZgWuskoERVOa31+CKvxwJjK3us+qiZlwuxYU2ZNH8H7s4W7uoRYnZIQoh6yt7k\n8C7wkHVMQkF1BiTK5uCgmHRTJGdz8pjwv614ODty8xUtzQ5LCFEP2ZscvIHLgM1Kqd8xbvcUpbXW\nr1RpZKJUjhYH3rs1inu+WMtzPyTRrJELfdo3MzssIUQ9Y29yeKnI6w6lrNeAJIca4uJoYdod0fz7\n9x10be1jdjhCiHrI3hHSThd5SPeZGubmbGHctZfj5erE2Zw8klJPmR2SEKIesSs5aK3zL/ao7kBF\n2Sb8tJUR01axfv9Js0MRQubRJeUAACAASURBVNQTFaqtpJQapJR6Qyn1kfX5quoKTNjvqYHt8PNy\nYfT0NWw9lG52OEKIesCu5KCU8lBK/QH8AjwN3GB9/lUptUgpJdOXmahZI1dm3hODh4sjd3y6muRj\nGRf/kBBClMPeK4c3gG7AGMBda+2HMWJ6jHX569UTnrBXkI87M++JQSl4fM5GmQtCCFEp9vZWuhEY\np7X+onCB1joXY7RzE+Ap4PFqiE9UQKifJzPujsHF0cE2qloIIS6FvVcOTSlSHO8CmwHfqglHVNZl\nAY0I9fNEa80ny/ZwIjPH7JCEEHWQvclhH3BNGesGASlVEo2oMnvSMpk0fwd3Tl9Delau2eEIIeoY\ne5PDNOAxay+lXkqptkqpeKXU+8Bj1vWiFgnz8+SD27uw7XA6d3/+J2dz8swOSQhRh9g7zuHfwGSM\nCqmLge1AAnAP8G+t9dvVFJ+ohH4d/Hl3RBTr9p3k/hky3agQwn72NkijtX5GKfUvIBZoglFfKVFr\nnVZdwYnKuyYygMycSMbN3czG/aeICW1qdkhCiDrA7uQAYE0E/6umWEQ1uTm6JT3a+BLY2M3sUIQQ\ndUSZyUEpFQf8pbXOtL4ul9Z6ZZVGJqpUYWL4Jekwa/Ye5+UhHaW7qxCiTOVdOSwHugNrrK/LGlWl\nrOssVRuaqA5bDp3mi8R9uDhZGDu4gyQIIUSpyksOA4Ct1tcDKTs5iDrk6avak5Gdx7Sle/B0ceTR\n/m3NDkkIUQuVmRy01ouKvF5YM+GI6qaU4uXrOpKZnc/bC3bi4eLI3T1lulEhRHH2Ft7bqZSKLGNd\nR6XUzqoNS1QnBwfFWzdGMDi8OUfPZJkdjhCiFrK3t1IbwLWMdW5AWNWEI2qKo8WBqSO74KDgwyXJ\ndGjuVWy60ZXJaSSlnuaB3vJPK0RDVJH5HMpqc4gCZBqyOsjioFBK4evhzJjP/2TqH7sAIzE8MmsD\nkUHeJkcohDBLeV1ZH8MojQFGYvhRKZV9wWZuQDPgm+oJT9SEq8Kb80FCMpN/38nuoxks3ZXG1JFR\nxIVJPUUhGqrybivtB1ZYXwcDScCFo6GzMXo0SW2lOszL1YnvH4qj/7+X8OPGQ3Ru2ZjOLRubHZYQ\nwkTl9VaaC8wFCvvCj9da762huEQN23o4nQKtCQ9sxMYDp3jxx838++bOZoclhDCJXQ3SWus7qjsQ\nYZ7CNob3b+tCXJgvn63Yy3uLdrEyOQ1fTxf8vVzxdncyO0whRA2yu7aSUsoRuApoT8meS1pr/YYd\n+3AFlgIu1mN/p7V+6YJtXIAvga7AceAWrXWKvXGKiktKPV2sjeGuHiG0b+7FXwdOMXfDQU5k5vLK\n0I4MjggwOVIhRE2xKzkopQIwvtTDMBqnC2suFO3BdNHkgNFG0U9rnaGUcgKWK6V+01qvKrLN3cBJ\nrXUbpdQI4C3gFnviFJemtO6qcWG+xIX5Et/Wj+d+SOLBmeu5qqM/E4eG49+orF7NQoj6wt6urP/C\n6K4aipEY4oB2GF/cuwG7ajBoQ4b1rZP1cWEX2aFA4VzV3wH9lRQAMk14oDc/PtSD5wZ3IGHHMa58\newk7/j5jdlhCiGpmb3LohTHZzwHr+1yt9W6t9fPAD8C/7T2gUsqilNoIHAUWaK1XX7BJYOFxtNZ5\nwGmMOayFSRwtDjzQO4x5j/fixi5BtGnmCUBOXoHJkQkhqovS+uL19JRSmcBVWuvlSqkzwPWFtZeU\nUv2BH7TWFRoxpZRqjNEb6h9a681Flm8GBmmtU63vk4GYCycVUkrdB9wH4O/v33XOnDkVOXytk5GR\ngaenp9lh2C09RzNh5TmubO3EwNaOWByq9uKurp2P6ibn4zw5F8VV5nz07dt3ndY6urR19jZIH+T8\nX+97MCq2FhbmiwYqXKBHa31KKbUYGARsLrLqINASSLU2gntjNExf+PlpWMdXREdH6z59+lQ0hFol\nISGBuvQzHDuTTdThTXy97QjbMt1484ZILm/RqMr2X9fOR3WT83GenIviqut82HtbaTHQ2/p6GvCM\nUupXpdR/gVcxbi1dlFLKz3rFgFLKDSPJbL9gs5+AO62vbwL+0PZc3oga5eflwsejujJ1ZBSHTp1j\nyNTlTJq/nfwC+acSoj6w98rhRaxXDlrr95VSzhg9iNyBd4CX7dxPAPCFUsqCkZi+0Vr/rJSaCKzV\nWv8EfArMUErtxpineoS9P4yoWUopro1sQY8wX175ZSt7jmVW+e0lIYQ57B0EdxSjAbnw/TsYSaFC\ntNZJGIX6Llw+vsjrLGB4RfctzOPj4czbN3cmN99ooE4+lsGMxH08NbAdXq4yeE6IuqgiVVnrn8RE\neOMN41lUmpPF+HVamXycLxJTuOqdpSzefrT8DwkhaqXyqrJWpJie1lrfXwXx1JzEROjfH7KywNUV\nFi2C2Fizo6oX7ujemssDGvHc90nc9fmfDO3cgvHXXk5TTxezQxNC2Km820pXU3yAmhfQCCgATgI+\nGFce6dZH3ZKQYCQGrY3nhARJDlWoa2sffn60J/+3OJn/S9hNYGM3nhnUweywhBB2Kq8qa1Dha6VU\nLPA18BDwrdY611r+4maMshk3V3egVa5PH+OKoTBBnJL5iqqai6OFJwa04+qIAFo2cQNg88HTNPFw\npkVjN5OjE0KUx942h3eAf2mtZ2mtcwG01rla65nAJODd6gqw2sTGGreSJk40Xk+eDD/+aHZU9VL7\n5l64OzuitebZ75MY8PYSZiSmUCDdXoWotexNDp2AHWWs2wFEVE04NSw2FsaNgwUL4IorYMQIWLbM\n7KjqLaUUH97elahWPrz43y3cMi2R5GMZF/+gEKLG2ZscjmAMSCvNcIp0c62TPDzg558hOBiGDIHN\nmy/6EXFpWjZxZ8bd3Zh0UyQ7/j7D4HeX8dcBuaUnRG1jb3J4F7hXKfVfpdTtSqkB1uefgDHAlOoL\nsYb4+sL8+eDuDoMGwf79ZkdUbymlGB7dkoVP9ea++FDCA42yXIUTDBW1MjmND5ckmxGmEA2aXcnB\nOujtQYwJeL4E5lufuwAPaq3rfnIAaN0a5s2DjAy46io4XqKkk6hCzbxc+edV7bE4KI5nZPPJsr2M\n+fxPEqxjIwpnqIsMqlBNRyFEFbB7EJzW+iOgFcaEPz2tz62sBfDqj4gI+Okn2LsXrr0WMjPNjqhB\ncHJ04JrI5mTlFjDmiz+ZuiGLh2euLzZDnRCi5lRohLTWukBrvVdrvdL6XD8L+vfqBbNmwZo1cMst\nkJtrdkT1XiNXJ964IZJZ98bg5erI2iP5ZOcVEBEoVw1CmKG8EdIjgXla6xPW1+XSWs+q0sjMdsMN\n8P778OCDcP/98OmnIBPS1QiLciCmuYW/jsOmg6eJC/PlnQU7CfXz4KqOzXF1spgdohD1XnkjpL8C\nugNrrK/Lo4H6lRwAHngADh82xkI0bw6vv252RPVaYRvD1NuiyDmwGeeW4TwyawPv3NKJn/46xN60\nTBq7OzEsKpBbu7Winb+X2SELUW+Vlxzacn5aULvmiK6XXn4Z/v7bKNAXEAD/+IfZEdVbSamnbW0M\nCQcgLsyXqSOjSEo9zaIne7My+Tiz/9zPV6v28dmKFN64IYJbu7UyO2wh6qXyymckl/a6wVHKuL10\n5Ag89hj4+8PNda9aSF3wQO+wEsviwnxtDdI92/rSs60vxzOy+WH9Qfq09wNg0bYj/LH9KLd2a2Xr\nFiuEqBx7J/tp2BwdYfZsGDgQ7rjDGBPRr5/ZUTVYTT1duLdXqO393rRMvluXyszV+wkPbMSIK1ox\ntHMLmUtCiEoor0F6F8WrspZHa63bV01ItZSbm9HFNT4err8eliyBqBLzFgkT3BMfyvCuLflx40Fm\nr9nPuB83M2v1fn59LN7s0ISos8q7cliN/cmhYfDxMQbJxcXB4MGwciWEhl78c6Laebs7cWdcMKNi\nW5OUeppT54zux1m5+dz+yWoGRwRwQ1QgPh7OJkcqRN1QXpvD7TUZSJ0RFGSU2ejZ0xhFvWIFNGtm\ndlTCSilFp5aNbe+PpGeRW6B55eetvPXbdgaFN2dEt5bEhjZFSddkIcrUsKcJvVSXXWYU6jt4EK6+\nGs6cMTsiUYbWTT3478M9+O2xeG7t1pKEHUcZ+fFqNh08DYDWcnEsRGkq1CCtlOoItAdcL1xX7wbB\nXUxsLHzzjdH+cOONRrJwllsWtdVlAY2YMDScsVdfRsKOo7aR1+P/u4VjZ7IZ0a0l8W39sDjI1YQQ\nYGdyUEp5A/8DehQusj4X/bOrYSUHMGovffwxjBkDd90FM2aAg1yM1WauThYGhQfY3jfxcOaXTYeZ\nt+VvAhu7cXN0S3Ly8+nRxrdYTaeVyWkkpZ4utbutEPWRvd9krwHNgX4YiWE4MBBj6tA9GCOpG6a7\n7oLXXjNqMT39tNnRiAp6YkA7Esf24/2RXQj18+CdhTvZ+XcGj8zawPJdaeTkFUh1WNEg2XtbaRDw\nKrDc+j5Fa70OWKiUmgY8DIyu+vDqiLFjjTIbb79tjKL+5z/NjkhUgIujhWsiA7gmMoADJ85icVCk\nHM/k/hnryM7NBxQP9AmVIoCiQbH3yqEFsFtrnQ9kAUWL2nwLXFfVgdUpSsGUKTB8uHH1MGOG2RGJ\nS9SyiTstGrsRF+bLdZ0CyMnXWBzgvUW76frKQu6cvoaj6VlmhylEtavINKGF/QP3ATFF1oVxvg2i\n4bJYjKTQr5/RBjFvntkRiUpYmZzGvM1HeLRfG9ycHHnpusu5M641JzJzbGMlZqzaxwcJyTIPtqiX\n7L2ttBwjIfwMzAQmKKVaAXkY04T+Uj3h1TEuLjB3LvTuDTfdBIsXwxVXmB2VqCBbdVhrEcDuYU1t\n71+45nLbdquSj/PLpsO8NW87YX4eDOzYnKvDA4iQtglRD9h75TARWGR9/S/gI+BGjHaG34BH7NmJ\nUqqlUmqxUmqrUmqLUuqxUrbpo5Q6rZTaaH2MtzPG2qFRI/jtN2Ng3NVXw86dZkckKqhodVgoXh22\nqPdv68LK5/oxYUhHArzd+HjpHj5etse2PjH5ODl59XM+LFH/2XXloLXeBeyyvs4BHrM+KioPeEpr\nvV4p5QWsU0ot0FpvvWC7ZVrray9h/7VD8+bGKOoePYxR1CtXGg3Vok64WHXYolo0duPOuGDujAvm\n9NlczmQbZTuSj2Vw68er8HJxpE+HZgy43J8+7f1oJMUARR1R5pWDUmq6UqpXVR5Ma31Ya73e+voM\nsA0IrMpj1Bpt28Kvv8KxY0YdptOnL/4ZUad5uzsR5OMOQGBjN6aPjuaayABW7k7j0dkb6PrKAlbs\nTgNkZLao/VRZv6RKqUyMkdD7gS+BGVrr3VV2YKWCgaVAuNY6vcjyPsD3QCpwCPin1npLKZ+/D7gP\nwN/fv+ucOXOqKrQq5fPnn0SMHcvpiAg2vfUWBWWMos7IyMDT07OGo6u96tP5KNCa5FMFrDuSx3Vh\nzng4Kean5LL6cB5d/C10aeZIC8/y7/DWp/NRWXIuiqvM+ejbt+86rXV0qSu11qU+AE/gLmAxkG99\nrADuBbzL+pw9D+u+1wE3lLKuEeBpfX01sOti++vatauu1b76SmvQ+qabtM7LK3WTxYsX12xMtVx9\nPx9z16fq6/6zTLd+9mfd+tmfdd/Ji/W/5m3TBQUFtm0+SNitV+w+prU+fz5W7D6mP0jYbUbItUZ9\n/92oqMqcD2CtLuN7tcw/V7TWGVrrz7TWfYFg4EXAB6Mx+rBSao5SarBSqkL1IpRSThhXBjO11j+U\nctx0rXWG9fWvgJNSquTN3rrktttg8mT47jtjNjm5pdDgXR8VyE+P9GTlc/2YOLQjLbzdSEo9basU\n+8myPWiteXjmelYmG7eiZKS2qEn2NkgfAF4HXldKdQNGATdjlNE4qpSaqbW+6LBgZfzmfwps01q/\nXcY2zYEjWmttPZYDcNyun6Y2e+opYy7qyZONxukXXjA7IlELtGjsxqjYYEbFBlNQYPzRkJWbz3uL\ndpGelYerkwOjp/9Ju8ZwYMl6Pri9S6kN40JUtQpPE6q1XgOsUUo9AbwBPGF92FMzogdwB7BJKbXR\nuux5oJV13x8CNwEPKqXygHPACOvlT9331ltGghg3zujRdPfdZkckahEHa0VYVycLa164ksTk4/y+\n9W9+2niIzcfziW/bhLgwX46kZzFl4U46BTUmMqgx7fw9cbRIwUdRtSqcHJRSbTCuHG7HuN2UDnxj\nz2e11su5yGhqrfVUYGpF46oTHBxg+nSjB9N99xljIa5r2JVHROlcnSz07dAMFycH5m85Qo8AxZqD\np1mZnIajgwO/JB1m9poD1m0d6NjCmwlDOhIe6E1OXgFOFiWTGYlKsbdktw8wAiMpdMMo1b0A46/+\nH7XWUmzGXk5ORttDv35w882waJEx7agQFyg6UjvnwGZGDwy3vf/rpYHsO36Wv1JP8deB0ySlnsLL\n1fjv/PXaA0yat53IoMZEBnnbngO8XSVhCLuVmRysDcfXYiSEwYAzsBV4DvhKa324RiKsjzw94Zdf\njEFy114Ly5df/DOiwSk6UjvhQPGR2nFhvgT7ehDs68HQzsWHCrVr5sk1kS1ISj3FtKV7yLO2ZSS9\nPJBGrk6s3nOcc7n5dApqLHNqizKVd+VwBPAGTgDTgC+0UaZbVAU/P2MUdVwc9OlDSP/+Rm2m2Fiz\nIxO1REVGahcVE9qUmNCmgNG4vfVwOslHM2yjsz9cksziHccAaNXEncggb2JCmnBHbHCJfX24JJnI\nIG+Z+KgBKq8VawlG/aQWWutHJTFUg5AQePNNOHaMVnPmQJ8+RqkNIaqIq5OFLq18GB7d0rbsvVuj\nmHVvDM8N7kDHFo3YsP8UP/11yLb+0dkbePrbv5ixah+eLhbpTttAlXnloLUeVpOBNFiHDoGDA6qg\nAHJy4PbbYc4c6NbN7MhEPeXl6lTiCiQrNx8wBsVm5eazfHca365LBcDRQTHm8z+5Lz6Ur1bv57Er\n2xIuEx/VexXurSSqWJ8+4OJCQXY2DhYLnDwJMTFGye/XXzdqNAlRzVydLAAopZg2KhqtNaknz5GU\najR27zt+lvf+2M3dPUN46b9beOm/Wwhs7Eb75l60b+7FNREBkjDqGUkOZouNhUWLSJk+ndAxYyA8\nHP79b2Ow3Ny5RpfX8eONcRFC1BClFC2buNOyiTs+Hk48MmsDj/Zrw4xV+3h6YDtQih1/n2HH32dY\nuvMYob4ehAd6s/VQOk98vdGWNDpYnwMbu0lPqTpGkkNtEBvL/uxsQgsbo19+GR58EF55BT76CL74\nwhhh/c9/GvNFCFFDypv46OG+bQDIySugwDpOtUBrgnzcWLfvZLF2jK/ujqFnW1+2HU5nzd4TtsTR\n2F16S9VWkhxqK39/mDoVHn/cGFH9yivwwQfw4otw//1GzyYhqll5Ex8VLnN2PN+vJTzQm09HG7Mf\nnsnKZeeRM2z/+wzhgcYfNct2HeP1X7fbtvdv5EL75o145+ZONPV04dTZHFydLLbbXIWk11TNk+RQ\n27VpYzRQ//Of8OyzRuG+KVPg1VdhxAhj1LUQ1eRSu9OC0fDdtXUTurZuYlt2b3woQzoFsv3vdNtt\nqd3HMmjkZnSzfWfBTr5avZ/gpu50aN7IdnsqMtDbdsUCxa9oRPWQ5FBXREfDwoWwYIGRJG67DSZN\nMuo1DRgAcj9X1AFKKZp7u9Lc25U+7ZuVWD8oPIBGbk5s//sMmw6e5pdNhwnwdiVxbH+mjoxizOd/\n0tRFk5b1J9d1asHuoxlk5xXQ17qvY2eycXe24O5skTaOSpLkUJcoBQMHwpVXwuzZxu2mq66C/v2N\nJNG1q9kRClEpsWFNiQ1ranufmZ3H3+lGdZ64MF9CfD3YdvgMzhb4bl0q361LJbq1jy05jPx4FbuO\nZuBscaCxuxM+7s70bOvLi9deDsBHS5JRChq7ORvrPZxp0diNwMZudsXXkG5vSXKoixwcjCuHm26C\nDz802iOio+GWW4zbTW3amB2hEFXCw8WRMD9jlrOVyWkcSc9mSJgTy/9WvDuiMx2aNyI3v8C2/T/6\nt+XQqXOcPJvDqcxcTp3LwdPl/Nfcx8v2kJaRU+wYw6ICeeeWzgB0e20hLk4O+Lg709jdGR93J/p1\naMbQzoEUFGgysnK5f8Y6nhvUgYEdm7PryBkemV0/b29JcqjLXFyMNojRo42ur2+/Dd9/bzRYv/ii\n0agtRD1wYRHCEf3Ci/WiKjSkU4ty9/PnC1eSmZPPycwcTp3N5eTZHJpY60vlF2iujgjg1NkcTp7N\n5dTZHFLSMgn1NZLTmew8pi5OBuCFHzfzwo+bUcCo2NbEhflyLiefBduOEOrrQYivBx4udfvrtW5H\nLwze3sbVw0MPwcSJxtXE558bjdhPPQVeXmZHKESlXKwIob2UUni6OOLp4kjLJsXXWRwULw/pWOZn\nPZwtLHyyFyfP5vL5ihR+2XSYiEBv+l1m/BGWfCyDR2dvsG3v38iFUF9PHu3fltiwpmRk53HsTDZB\nPm441YH5NyQ51CcBAUZ31yeeMGaamzDBeD9+PNx7LzhLn3JRN1Wm11RVcbQ40KaZFyuT00jcc5xH\n+7Xhq9X7cbIYDd/t/L2Y93g8e49lsictkz3HMtmblmHrK7Iq+Tj3fLkWRwdFq6buhPp6EOrnyajY\n1gT5uJOXX4DFofbMwyHJoT5q1w6+/RZWrzZ6Nj3yCLzzDrz2GgwfLt1fhbhE5Q0KjAvzpUPzRnRo\nXvpA1Y6BjZh0UyR7bYkjk6W70rg5OgiA2Wv289a8HYT4ehDqZ9yaCvH1YODlzXFzrvlxH5Ic6rOY\nGFi8GObNM5LEiBHnu7/27292dELUOfYMCixLgLdbseq4YLRzFF4ntPP34sYugexJy7SNMNcaNk+4\nCjASQsKOo4T6eeIATP1jN68M7YgP1TPuQ5JDfacUDB5sdIGdOdNoqL7ySuP9m29CVP3rZSFEdanq\n21sWh/O3kIrOwwFGpdwDJ87aelu5O1vIzdf8tukwJ8/mAvDP75K4JsSR5ctKNs5XliSHhsJigVGj\njKlJ/+//jFtMXbrAyJFGY3ZoqNkRCiGKcHWy0Nb/fGeSUbHBjLJOyHQyM4c9aZnMXL2PH9Yf5NF+\nbaq8/UVuPjc0rq7w5JOQnAxjxxqVXzt0MLrE/vorvPEGJCaaHaUQohw+Hs5k5+WTsOMYQ8Kc+Gr1\nftuETFVFkkND1bixMV/Erl3GOIn//Aeuucbo5dS3LyQkmB2hEKIMRdsYbmjrzNSRUTwya0OVJghJ\nDg1dYCBMm2ZcOQBoDdnZRr2mwYONXk6bNxvLhRC1QnkN41VF2hyE4eabjbkjcnKM9okhQ4yk8OST\nxvqAAKMRe8AAo0FbRl8LYZqaGPchyUEYrDPSkZBgTF1aOPHQ/v1GJdgFC+B//zMmHgLo3NlIFAMH\nQs+eRluGEKLekOQgzouNPZ8UCrVqBXffbTzy82HDBvj9dyNZTJlijJtwdYVevYxEMXCgMdVpLRnl\nKYS4NDXa5qCUaqmUWqyU2qqU2qKUeqyUbZRS6j2l1G6lVJJSqktNxijKYbEY1V+ff94YXHfiBPzy\ni1Ho78ABo5ZTZCS0aGF0m/3qK/j7b7OjFkJcgpq+csgDntJar1dKeQHrlFILtNZbi2wzGGhrfcQA\nH1ifRW3j6QlXX208AFJTjSuK3383usXOmGEsj4w8314RHw9u9tXOF0KYp0avHLTWh7XW662vzwDb\ngMALNhsKfKkNq4DGSqmAmoxTXKKgILjrLmMioqNHYd06Y9xE06bw3nvGxEQ+PkaimDQJ/vpLekEJ\nUUspbdJ/TqVUMLAUCNdapxdZ/jPwptZ6ufX9IuBZrfXaCz5/H3AfgL+/f9c5c+bUUOTVIyMjA09P\nT7PDqDYO587ROCkJn7VrabJ2LR4pKQDk+PhwsmtXTkRHczI6mpymTWm0ZQtua9Zwrls30juWXUK5\nIanvvx8VIeeiuMqcj759+67TWkeXulJrXeMPwBNYB9xQyrqfgZ5F3i8CosvbX9euXXVdt3jxYrND\nqFmpqVp/9pnWI0dq7eentXENoXVoqNaOjrpAKa1dXLSeN8/sSGuFBvf7UQ45F8VV5nwAa3UZ36s1\n3ltJKeUEfA/M1Fr/UMomB4GipQuDrMtEfRIYaIzMHj0aCgqMW0wLFhgD8vLyjEqV2dkwaJAxmjss\n7PyjTZvzr1u0kBLkQlSDGk0OypjF4lNgm9b67TI2+wl4RCk1B6Mh+rTW+nBNxShM4OBgVIeNijIa\nrPv3R2dnoxwd4b77jC60yclGG8b33xvvC7m6QkhI8eRR+AgJkQmOhLhENX3l0AO4A9iklNpoXfY8\n0ApAa/0h8CtwNbAbOAvcVcMxCjNZB+PtnT6d0DFjSo67yMszBuYlJxuP3bvPv/7jDzh79vy2Dg7Q\nsmXpiSMsDBqVPimLEKKGk4M2GpnLHR1lvQ/2cM1EJGql2Fj2Z2cTemFiAHB0NMqLh4YaXWOL0hqO\nHDmfLIo+fvwRjh0rvr2vb/FbVEUf/v6walXJEeNCNBAyQlrUH0pB8+bGo0ePkuvT00tPHMuXG91v\nCwrOb+vqarR5aG0M/iu8igkKOv/w8ip5DCHqCUkOouFo1Oh828aFcnIgJeV8wpg9G1auNNbl58PH\nHxuPory8zieKwMDiiaPwfdOmUkpE1EmSHIQAo+G6XTvjAdC1qzHPdk6Ose7XX406U6mpcPCg8Vz4\nOHgQtm6Fw4eLX30AuLiUnTgKXzdvblydCFGLSHIQojRlVaktbzrVvDyjzePCxFH4OjHReJ+TU/xz\nFotREr2sJPL334T8+KPR2N6jBzg5GQnLYqm5q5LERGl/aWAkOQhRltKq1JbH0dH4Qg8MhJgyyoFp\nDWlpZV+BbNkC8+ZBZmaxj7UGmDWr5P6cnc8ni6KvL3y+1HVOTkZc779vJD9HR3juOWjf3ugNZrGc\nf5T3vqrW/fknrWbMwfBCFAAACAVJREFUMJbHxRnJ0cGh5m/dVUWy1Nq4ZZmfb1xxXvi6tGUXrl+/\nnpB584wr1CpO2pIchKhJSoGfn/Eore0DjC+N9HTjS3nyZPj8c2OZg4MxKLBXL+PqIze39Oey1p07\nZ+y3vG0Kn4uOJSkqNxdeeaXaTo89QgE++6zkCqWKJ4sLn8tbV5HnrCyjCrHWxrKAACOR2vuFXris\nirQCYy74RYuqNEFIchCitlEKvL2Nx733wpw5FGRn4+DiAuPG1cxtnfx8IxHk5sKKFTBsmPHayQm+\n/BI6dSr7i7C8L8mKrLvw/cKFxlVV4Zdy//7GoEmtje3sea7ItmU9b9lijLUp5OdnVB6+2JVP4evK\nri98/dNP8PXXqIICI6knJEhyEKLBsLZ9pJQ1KLC6FH4JuboaVyt//GF+m0NsLCQknE+UEyeaE0ti\nYvHOCh98YE4crVvDjz8a58PZ2fi3qUKSHISo7cobFFiDMZjeEG1WoiwjjlqRLKvxfEhyEELUHbUh\nUVrjMD1ZWuOorvMh5SyFEEKUIMlBCCFECZIchBBClCDJQQghRAmSHIQQQpQgyUEIIUQJyphbp25T\nSh0D9pkdRyX5AmlmB1GLyPkoTs7HeXIuiqvM+WittfYrbUW9SA71gVJqrdY62uw4ags5H8XJ+ThP\nzkVx1XU+5LaSEEKIEiQ5CCGEKEGSQ+0xzewAahk5H8XJ+ThPzkVx1XI+pM1BCCFECXLlIIQQogRJ\nDkIIIUqQ5GAypVRLpdRipdRWpdQWpdRjZsdkNqWURSm1QSn1s9mxmE0p1Vgp9Z1SartSaptSqhbU\niTaPUuoJ6/+TzUqp2UopV7NjqklKqelKqaNKqc1FljVRSi1QSu2yPvtUxbEkOZgvD3hKa3050B14\nWCl1uckxme0xYJvZQdQS7wLztNYdgE404POilAoEHgWitdbhgAUYYW5UNe5zYNAFy54DFmmt2wKL\nrO8rTZKDybTWh7XW662vz2D85w80NyrzKKWCgGuAT8yOxWxKKW+gF/ApgNY6R2t9ytyoTOcIuCml\nHAF34JDJ8dQorfVS4MQFi4cCX1hffwFcXxXHkuRQiyilgoEoYLW5kZhqCvAMUGB2ILVACHAM+Mx6\nm+0TpZSH2UGZRWt9EP6/vXsLlaqK4zj+/aFRqRV285JUPpQRPWgPIgndTAmRoocupKIRYdEVyocU\nkkJMKqIeSrGHTI8lZlLhk4QlQUE3xKTyQokeO0fTolDJTP89rDU07n1OJ21gT+7fBxZn9t5r1l4M\nnPnPWmtm/XkB2Al0Ab9GxLpqe9UWhkREV37cDQxpRaMODm1C0iDgHeCxiPit6v5UQdIUYG9EfFl1\nX9pEf+BqYFFEjAEO0qIpg/+jPJd+KyloDgcGSppWba/aS6TfJrTk9wkODm1A0mmkwLAiItZU3Z8K\njQdukbQDWAncKKmj2i5VqhPojIjGSHI1KVjU1U3ADxHxU0QcAdYA11Tcp3awR9IwgPx3bysadXCo\nmCSR5pS/jYgXq+5PlSLiyYgYERGXkhYa10dEbT8ZRkQ3sEvSqHxqAvBNhV2q2k5gnKQB+f9mAjVe\noG/yPjAjP54BvNeKRh0cqjcemE76lLwxl8lVd8raxsPACkmbgNHAgor7U5k8gloNfAV8TXr/qtVW\nGpLeAj4FRknqlHQvsBCYKGkbaXS1sCX38vYZZmZW5JGDmZmVODiYmVmJg4OZmZU4OJiZWYmDg5mZ\nlTg4WC1JmikpeimV7V8kaamkzqrub9bQv+oOmFXsdtIvkZv9WUVHzNqJg4PV3caI2F51J8zajaeV\nzHrRNPV0raR3JR2QtF/SK5LOLNQdJmmZpH2SDkva1NOmcJJGSlouqTvX+17Syz3UGyPpY0mHchKX\n+wvXh0p6Q9KPuZ0uSWslXdj6V8LqyCMHq7t+OTdAs2MR0bxleAewCngVGAs8BQwEZgLkbbQ3AIOB\nOcAuYBqwXNKAiFiS640EPgMO5Ta2ARcDkwr3Pxt4k7R9+TPAPcAiSVsi4sNcZzlwCTA7328Iaa+h\nASf7QpgdJyJcXGpXSG/s0UtZW6izuPDcucBR4PJ8/FCud32h3gekHTL75eNlwAFg+D/0a2lu64am\nc6cD+4ElTecOAI9U/Tq6nLrFIweru9soL0gXv620qnC8EphPGkVsJWVr2x0RHxXqdQCvA1eSNoqb\nRAo8fWUvOxR/jxCIiMOStpJGGQ2fA7Pz7qTrgc0R4Y3SrGUcHKzuNkffC9J7ejlupHM9l5SZrKi7\n6TrAeZQDUU9+6eHcYeCMpuM7gXmkrHkvAV2SFgPz4/gpMbOT4gVps74V0y42jnfnvz8DQ3t43tCm\n6wD7aFF+8IjYGxEPRsRFwBWk6aingVmtaN/MwcGsb3cUju8i5bhuZGjbAIyQNL5Q727SmkMjQc86\nYEoja1erRMSWiJhDGnFc1cq2rb48rWR1N1rS+T2c/6Lp8WRJz5Pe3MeSpnOWRcS2fH0p8CiwRtJc\n0tTRVGAiMCsijuZ684DJwCeSFgDbSSOJm+MEMt5JOoe02L0C+A44QsqtPDj30ew/c3Cwunu7l/MX\nND2eBjwOPAD8AbwGPNG4GBEHJV0HPEfKwnUWsAWYHhEdTfV2SBpHWsx+FhhEmpo60bSOv5Oyod1H\n+jrrsXy/qRHRkhSRZs4EZ9YLSTNJ3za67F8sWpudUrzmYGZmJQ4OZmZW4mklMzMr8cjBzMxKHBzM\nzKzEwcHMzEocHMzMrMTBwczMSv4CVCeoNb4pd5sAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-xNZnc4JGJY8",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 284
        },
        "outputId": "831ab21f-c28c-47be-fd32-0f3be3523352"
      },
      "source": [
        "plt.plot(epoch_record, bleu_score_record_post, 'x--', label='Post-LN (Adam w/ warm-up)')\n",
        "plt.plot(epoch_record, bleu_score_record_pre, '.-', color=\"red\", label='Pre-LN (Adam w/o warm-up)')\n",
        "# plt.title('')\n",
        "plt.xlabel('Epochs', fontsize=16)\n",
        "plt.ylabel('BLEU', fontsize=16)\n",
        "plt.legend()\n",
        "plt.grid()\n",
        "plt.show()"
      ],
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYMAAAELCAYAAAA7h+qnAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0\ndHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deVhV1frA8e9iFnBAVJwFcSgHBCQH\nFBU1tSyH23DLBm3Orrfh17UsK22uW5lNZt7s2mBaDg12U1OSnDVxnkUlQ0xFBUFmzvr9seHIUVCQ\nw9kHzvt5nvNw9j777P2yxP3uvdbaaymtNUIIIVybm9kBCCGEMJ8kAyGEEJIMhBBCSDIQQgiBJAMh\nhBCAh9kBXKkGDRro4OBgs8OolHPnzuHn52d2GE5DyuM8KQtbUh62KlMeCQkJqVrrhheur7bJIDg4\nmE2bNpkdRqXEx8fTr18/s8NwGlIe50lZ2JLysFWZ8lBK/VHaeodWEymlfJRSG5VS25RSu5RSLxat\nD1FKbVBKJSqlvlFKeTkyLiGEcHWObjPIBfprrbsA4cAQpVQP4E3gXa11G+AMcJ+D4xJCCJfm0GSg\nDZlFi55FLw30B+YXrf8cGOHIuIQQwtUpRw9HoZRyBxKANsBHwFvA+qK7ApRSLYDFWutOpXz3QeBB\ngKCgoK5z5851WNxVITMzE39/f7PDcBpSHudJWdiS8rBVmfKIjY1N0FpHXbje4Q3IWutCIFwpVQ/4\nDriqAt+dAcwAiIqK0tW9QUkaxWxJeZwnZWFLysNWVZSHac8ZaK3TgBVAT6CeUqo4MTUHjpoVlxBC\nuCJH9yZqWHRHgFKqFnAtsAcjKdxctNlo4AdHxiWEENXGunW0nD0b1q2z624dXU3UBPi8qN3ADfhW\na/2TUmo3MFcp9QqwBZjp4LiEEMI55efDn39CUhIsXw5vvUWIxQKzZ0NcHPTsaZfDODQZaK23AxGl\nrD8EdHNkLEII4RQKCuDoUeNkn5QEhw/bvk9OBovF5isKIC8P4uOrZzIQQlQjS5fCjz9Cv35w7bVQ\npw64yXBmFWaxQEpK6Sf6pCTjqr+g4Pz2SkGzZhAcDH36QEiI8T44GM6cgbvuwpKbi5uXl/FvYyeS\nDIQQts6cgSeegM8/N5anTTN+urlB/frnX4GBtj/Lel+njnGCq6ksFjh+vOyT/R9/GFU9JTVpYpzc\ne/aE22833hef9Fu0AG/vso/XtClJn31G63vvtdtdAUgyEEIUO3sW3nsP3nkH0tPPr3dzgyFDICIC\nTp82XqdOwbFjsHOnsZyRUfZ+3d0vnSzKWufvbySR4gZTb+8rO/lpDYWFxtV3YeH51+WWL1y3ZQus\nXAkBAUYCKD7pJyVBbq7tMRs1Mk7uXbvCTTfZXt23agU+PhX/PYr17MmR3Fxa2zERgCQDIURWFnz0\nEbz5pnGSHz7cOIE99JBRL+3lBc89d+kTcX7++URRnCxK/iz5/uhR2L7deJ+ZWfY+PTygdm1ISyNE\na5g5E1q2NOKpyIm9Kh6srVMH2rWDzp3hxhvPn+xDQoyTva+v/Y9ZxSQZCOGqcnNhxgx49VWjmmPI\nEHjpJbjmGuPzNm2MBsp+/S5/Re7pCUFBxqsi8vIuThYl38fFwcaNRoOp1sZJuFMn426j+OXhcell\ne2wzfz58+aVxR+DuDhMmwDPPVLzMnZgkAyFcTX4+/Pe/8PLLRk+Vvn2Nk13v3rbb9exp1zrpUnl5\nQePGxqs069bBgAFGg6m3N3zySdXHVJrAQPj22/N3SjXwaWhJBkK4isJCo2/6iy/CoUPQowfMmgX9\n+ztvA2/PnhAXVyUNplcSR7nvlKohSQZCFFu7Fv73Pxg6FKKjzY7GfiwWmDcPJk+GvXuNhuD//Q+u\nu855k0BJVdRgeiVx1MQkUEw6DQvXlpZmnCiHDoVeveC114zqklGjjJ4jJft/Vzdaww8/GCf/224z\n6roXLoSEBLj++uqRCITDyJ2BcC1aw9atsHix8Vq3zqg+KdnVT2v45huYM8eoKx46FIYNg0GDjN4t\nzk5r44Gx55+HTZugbVv4+mu49VYjIQhRCkkGouY7cwaWLTNO/kuWwF9/GesjI41eIdddZ1SlDB58\nvoHwxx+Nu4Yff4SffoIvvjDW9+9vJIYbb4Tmzc39vUoTH290A12zxujq+NlncNddRs8YIS5B/kJE\nzWOxXHz1b7EYDwsNGmSc/AcPvrgHS2kNhDffbFQVrV1rVLn88AM88ojxiow0EsOwYRAebm61y7p1\nxp1AXBw0bQoffwz33mskMCHKQZKBqBnOnIFffjl/9X/8uLG+a1d49lkjAXTrdukr5LIaCD08jDFi\n+vSBt982GmF//NF4vfii0TDbooVxtzB8uNFV81LDCdjT5s1GEvj5Z+Op13ffNR4Wq1XLMccXNYYk\nA1E9WSzG8ADFV//r15+/+h88+PzVf0UfgrocpeDqq43X00/DiRNGz5wffzS6aU6bZrQrDBli3DFc\nf70xvIK97dwJkyYZDcIBAfDGGzBuHPj52f9YwiVIMhDVx+nT56/+ly69sqt/e2vUCO65x3hlZ8Ov\nv56/a5g3z2iw7d37fHVSmzaVO97+/cbdyJw5RtKZPBkefxzq1rXLryNclyQD4byKr/5//tlIABs2\nGOvq17et+7f31f+VqlXL6Hk0dKhRZ5+QcD4xPPmk8br66vOJoXv38vfuSUoyhor44gujCurpp+Ff\n/zJ6OwlhB5IMhHMoHpkyN9eo/y+u+z9xwvg8KgomTjx/9e/sXSTd3Iwxfq65xhj24fBhWLTISAzv\nvGMMCteoEdxwg5EYBg4svYrn6FFj7KBPPzX2+eijRiJwlgQoagxJBsJ8c+bA3XcTUlBgnPTAuPov\nWfffqJG5MVZWSIhxIn/0UaPL6pIlRmJYsMDo/unjYySEYcOgcWOCv/nGuLv44QfjbuiBB4yqsGbN\nzP5NRA0lyUCYY+9eo07922+NxlCKpvJTyugN8+GHzn/1f6Xq1TOeCL7tNmPQuFWrjMTwww/GMw1A\nq+Jtb7gBPvjAeGZAiCokw1EIx9m3z6gyCQsz6s4nTTJ6wjzxBPj4YHFzM66Q77675iaCC3l6Gg+y\nTZ1qDB732GOglJEY3d2NMZIkEQgHkDsDUbX27Tt/B7Bjh3Hl36sXvP++MYFK06bGdrfcYv7IlGZT\nCv7+d5gxo0rmuBXiUiQZCPvbv/98Ati+3VjXq5cxpeJNN5Ve7+0sI1OazVmGbBYuR5KBsI8DB84n\ngG3bjHXR0Ub1x003Oec4Ps5KEqMwgSQDceWKE8C8ecZYQGAkgHffNcb0kQQgRLUhyUBUTGLi+QSw\nZYuxrmdPmDLFSAAtWpgbnxDiikgyEJd38OD5KqDiBNCjhyQAIWoQSQaidIcOnU8Amzcb67p3N56e\nvflmaNnS3PiEEHbl0GSglGoBfAEEARqYobV+Tyk1GXgAOFm06bNa658dGZvAGDKhOAEkJBjrunUz\nhm2++WZo1erS3xdCVFuOvjMoAJ7UWm9WStUGEpRSy4o+e1dr/baD4xHffw+ffGIMhLZ3r7GuWzd4\n6y0jAcgDT0K4BIcmA631MeBY0fsMpdQeQAZbMcv77xtPvILxwNM//mGMrBkSYm5cQgiHU1prcw6s\nVDCwEugE/B8wBjgLbMK4ezhTynceBB4ECAoK6jp37lwHRVs1MjMz8ff3N+XYDVatosPkySiLBQVY\n3NxIuvdejtxxhynxgLnl4WykLGxJediqTHnExsYmaK2jLvpAa+3wF+APJAB/K1oOAtwxxkp6Ffjs\ncvvo2rWrru5WrFhhzoE//1xrd3etO3bUulYt432tWlqvXWtOPEVMKw8nJGVhS8rDVmXKA9ikSzmn\nOrw3kVLKE1gAzNZaLyxKSMdLfP4f4CdHx+UyPvrImB6xf39jlMwdOy6eBF4I4XIc3ZtIATOBPVrr\nKSXWN9FGewLASGCnI+NyCVrD668bE8QMHw5z5xojhJY1CbwQwqU4+s6gF3AXsEMpVTR+Ac8Ctyul\nwjG6myYBDzk4rppNa5gwAf79b7jzTmMyFU9Ps6MSQjgRR/cmWk3RHCYXkGcKqkphodFL6JNPYOxY\nY9IYN5nGQghhS84KNVl+vjFRzCefGHcGH30kiUAIUSoZjqKmys6GW281plF8/XUjGQghRBkkGdRE\nGRlGI3F8PEybZlQPCSHEJUgyqGlOn4brrjPGFvrySzDxITIhRPUhyaAmOXYMBg0yJp1ZuBCGDTM7\nIiFENSHJoKZISoKBA+Gvv+B//4MBA8yOSAhRjUgyqAn27jUSwblzsHy5MfGMEEJUgCSD6m7zZhg8\nGNzd4bffICzM7IiEENWQdDqvzlavhthY8PWFVaskEQghrpgkg+pq6VKjsbhxYyMptG1rdkRCiGpM\nkkF1tGAB3HgjtG9v3BHIhPRCiEqSZFDdzJplPFl8zTWwYgU0amR2REKIGkCSQXXy/vtwzz1Gt9Ff\nfoF69cyOSAhRQ0gyqA60hpdfNuYrHjkSFi0CPz+zoxJC1CDStdTZaQ3jx8M77xgjkM6cCR7yzyaE\nsC85qzizwkJ4+GH49FNjqsr33pMhqIUQVULOLM4qL88YZO7TT42pKt9/XxKBEKLKyJ2BM8rOhptv\nhp9/NqaqHD/e7IiEEDWcJANnc/asMdroypXGDGUPPmh2REIIFyDJwJmcOgVDhsDWrTB7Ntx+u9kR\nCSFchCQDZ5GSAtdeCwcPwnffwQ03mB2REMKFSDJwBocOGUNQnzwJixcbg88JIYQDSTIw2+7dxh1B\nTg7ExUG3bmZHJIRwQdJX0Szr1tHmvfegZ0+wWIy5CCQRCCHKMP23g6w9mGqzbu3BVKb/dtAu+5dk\nYIZ16yA2lmbffw8ZGcYzBJ06mR2VEMKJhTWvy7ivt1gTwtqDqYz7egthzevaZf9STWSGX36B3FwU\nGA+SJSaaHZEQwgnlFhSSnp2Pv7cH0aENeHl4R8Z+lUDfporVq7bw4agIokMb2OVYkgzMcPQoANrN\nDeXlBf36mRuPEKJM0387SFjzujYn3bUHU9menM7DfUMv+32tNRYN7m6KnPxCNh85Q3pWPmnZ+aRl\n5ZOWncegDkF0bVWfA8czeHTuVtKz8kjLzicrrxCAD26P4MYuTQnw8yK0oT8/Hkzj0f5t7JYIQJKB\n4504AXPmQN++HG7bltb33mu0GwghnFJx9cyHoyLo2TqQH7am8MIPO3kktg0/bkshPSuPdkG16d46\nkIycfJ74Zhvp2XlFJ/p80rPyefzatjzSrw0nM3IZ9Z8NNvv3dFe0CPCla6v61PJyp1m9WnRsWod6\ntTyp5+tJXV8vOjczqoLy8i0cSj3HsFBPvtpwhB6hgdXzzkAp1QL4AggCNDBDa/2eUqo+8A0QDCQB\nt2qtzzgyNod56SVjuIkZMziSkkJrSQRCOKVCi2Z3yll2JKdza1QLxn29hTu6teSDFUa17huL91q3\nHRMdTPfWgXh7uJN8JosAXy/aNPI3Tua1vIhoEQBAUB0f5jzQg7pFJ/p6vp7U8nRHKQVA8wBfPh0d\nVWo8aw+m8n/ztjHtjkjy/tzJbf07WZOUPRKCo+8MCoAntdablVK1gQSl1DJgDBCntX5DKTUBmAA8\n7eDYqt7+/cYQEw89BO3aGQ+aCSGcyre//8nyPcdZf+gUZ3MKALixS1Pu7N6S939NZHDHIG7p2sJ6\nMq9by4u6tTwB8PJwY8njfcrct5eHGz1DA68oru3J6dYTf/yfEB3agA9HRbA9Ob36JQOt9THgWNH7\nDKXUHqAZMBzoV7TZ50A8NTEZPPMM+PjACy+YHYkQLk9rzZHTWaw9eIpDJzOZOLQDAL/sPs7+4xlc\n37kJPUMD6dk6kMSTmYz7eguP9m/DVxuOMDo6mKjg+g6Nt7T2iejQBnarJlJaa7vsqMIHVioYWAl0\nAo5oresVrVfAmeLlC77zIPAgQFBQUNe5c+c6LN7KqrNzJ5H//CeH77mHP+6+G4DMzEz8/f1Njsx5\nSHmcJ2Vhy57lsf9MIauSC9h9qpBTOcb5r5634o2YWvh4KHILNd7uyrr9nlOFTNuawyPhPlwd6H7R\nshkqUx6xsbEJWuuL66K01g5/Af5AAvC3ouW0Cz4/c7l9dO3aVVcbFovW0dFaN26sdWamdfWKFSvM\ni8kJSXmcJ2Vh60rLIzUjR/+0LUU/u3C7PnLqnNZa6zkb/tDhLy7VY7/apL9Yl6QTT2Roi8VS5j4+\njk/UaxJP2qxbk3hSfxyfeEUx2UNl/j6ATbqUc6rDexMppTyBBcBsrfXCotXHlVJNtNbHlFJNgBOO\njqtKff89rF0LM2bI3MXC6VW2K6XZjp/NYcbKQ6xJTGXvXxkA+Hm5c22HIFrU9+Vvkc25NaoFbm7q\nMnsyVHX1jLNw6BPIRVVAM4E9WuspJT76ERhd9H408IMj46pS+fkwYQJcfTXcc4/Z0QhxWVX9pGt5\nlWf4hey8QlYfSOXfS/by03ajQ4a7m2L2hj8I9Pdi/OD2LHwkmq2TBtGvfSPAaMQtbyJwJY6+M+gF\n3AXsUEptLVr3LPAG8K1S6j7gD+BWB8dVdT791OhF9OOPMpG9qBaiQxswLrYNd8/ciKe7G7kFhTQP\n8GXKL/tpdmstWgX6se7gKZbu+gs/b3d8vTzw9/bA18ud6zs3wc/bg+Nnczh9Ls+63s/bA28PN2sX\nyvIo2b8fzielD0dF8NGKRFYdOMnmP9LIK7Tg4aa4r3cIN4Q1pYG/N9snDcbLQ0bbqQhH9yZaDZT1\n1zDAkbE4REYGTJ4MffrI/ATCqeUXWvht30mCG/jSplFturSoSx0fD05n5dM+yJ+WgX6cyy3A0904\nwR48mcnCzcmcyyuk0HK+E0qfdg3x8/ZgzsYjTF1+wOYY7m6KLS9cSx0fT/6z8hA/bU/Bz9ujKJkY\nCePl4Z1wc1OsTUzlj9NZjOrWgge/SKCln4VDcb/z2ZhriA5twJRf9pNbYOGeXsH0DA3kmuD6+Hmf\nP51JIqg4uVStSm+/bTxxvGgRVOCKSAhH0Fqz8+hZFmxOZtG2FE6dy+PeXiG8cGMHcvMtoJS1K+Wk\nYR1t6sjv7NGKO3u0QmtNboGFc7kFnMstJNDPCzD65bcLqs253AKy8grJzC0gK68AX0+j942/jwf1\nfL04l1vAmaxszuUWkFdg4dWRxv+T+ZuTWbj5qPV4u3Ohob8XPUKMPvpzHuxhTUzCPiQZVJVjx4xk\ncOutMjS1cDpaa26Zvo5Nf5zBy92NgR0a8beI5vRt39Cojplz/snWHqGBZT7pqpTCx9MdH093Akv0\ndAxt6E9ow7K7Pt7erSW3d2tZ5ucvD+/EU4OvYvWBk7z00256BMGmVFh/+BTRoQ0kEVQBSQZVZfJk\no/H4tdfMjkQIsvIKWLrrL9YknuKtm8NQSjGkU2NGRjbjhs5Nqevrad225JOuYP8nXcvDz9uDbclp\nvLZ4L9Pv6krenzsZ08K+wy8IW5IMqsKePUbD8bhxEOr8XfFEzWSxaNYfOsWCzUdZvPMYWXmFtKhf\ni5MZuTSq48P9Ma1L/Z6zdKWs6uEXhC1JBlVhwgTw94fnnjM7EuGCCi0adzfFsj3HeejLBGp7ezA8\nvCl/i2xOVKuACvXoMZOzJCVXIcnA3lauNLqRvvYaNGxodjTCRZzKzGXRthQWbjnKoA5BjOvflr7t\nGvLhqAgGXh2Ej6c5wyaI6kOSgT1pDePHQ7Nm8NhjZkcjXMAvu/7i203JxO87QYFF06FJHZoF1ALA\nx9OdG8KamhyhqC4kGdjT/PmwcSN89hn4+podjaiBtNbsP55J+8a1Afh2059sT07nvt4hjIxsxlWN\n65gcoaiuJBnYS16eMUR1585QNCqpEBVxqTGBhnZuwndbjvLdlqMcTj3HqqdiaVHflzdvCqOerxfu\nMryCqKTLJgOlVP9LfFwAHNda77NfSNXUJ5/AwYOweDG4S/2sqLjShl8Y+9VmmtT1sc6q1bN1II/0\nCyXQ33i4K9Df27R4Rc1SnjuD5RhTVF546WF9Bl0pdQx4Rmv9pR1jqz7S043pLAcMgMGDzY5GVFPR\noQ14eXhHHvhiE10CYe+qLbx5U2emLNvP+MHtGR7elOYBUv0oqkZ5kkHsJT5zB5oCNwOzlFJntNY/\n2SWy6uTf/4bUVONnNem2J5zHzzuOsepAKhsOneJQ6jkA1qbAo/1DGNKpCUM6NTE5QuEKLpsMtNa/\nlWM/XymlvgXGA66VDJKTYcoUuOMOiIw0Oxrh5E6czWH94dMcS8vmoaJ+9DNXH2b/Xxl0C6lPz9BA\nFm1LoW9T+GrDEXqEBkq/euEQ9mxAngPMsuP+qodJk8BigVdeMTsS4aTWHzrFD1tTbK78A/28uK93\nCB7ubnx8ZySBft5sOHyKcV9vsQ6/cPsAGX5BOI49k0EG4GXH/Tm/HTtg1ix44gkIDjY7GuEEiq/8\n1x86xfhB7Qnw82Lrn2n8tC2FbiH1ub1bS3q0DqRD0zrWHkCNavsAMvyCMJc9k0F3jIlpXMeECVCn\nDjz7rNmRCBMlpZ5jxqpDrD90ikMnjSv/2t4e3BTZjK5+9RndM5gHYlpftvunDL8gzFTpZKCU8gCG\nAU8DUysdUXXx66/w88/w1ltQv77Z0QgHKXnl369dQwZ1bEyBRbNoawrXhNTntmtaGFf+TergUTTM\nci0v6WosnF95njP4kxLdSC/gDjQo2s9y4FX7hebELBZ46ilo2dIYmVRUa5ebAD6/0MKkH3fZXPn7\ne3vQuoEfAKEN/djywrXWk78Q1VF57gziKDsZFAAngBVa6+V2i8rZffMNJCTAl1+Cj4/Z0YhKKvmw\nV5tG/nyx7g/+s/IQ/YsmUPd0d2NHcjrBgX6lXvkrpfBwly7FonorT9fSMQ6Io/rIzTXaCMLDYdQo\ns6MRdhAd2oChnZtw16cbKCy67PHxdLOp3vlxXK9qM/SzEFdCxiaqqGnTICkJli0DN6kWqG4KLZod\nR9NZfeAkG5PO8OndUXh5uBHg60nTgFr8eTqbW6Oa89rIzjbVPpIIRE132bOZUmqzUqpjiWWllHpf\nKdX8gu2ilFJnqyJIp3HmDLz8sjHkxMCBZkcjKmB7chqPzE4g8uVljPhoDW//sp9TmbkcP5sDQI/Q\nQM7lFvJo/zYs33OCjUmnTY5YCMcqz51BOOBXYtkN+AfGA2bJJda7X7BdzfP665CWBm++aXYk4hLS\ns/NZdzCVVQdSGR7ejG4h9cktsLD1SBqDOwbRu21DeoUGWgd5W3sw1ebhrktNAC9ETXWl1USud8/8\nxx/w/vvG8NRdupgdjbhAVl4BH8cfZNWBVLYnp2HRRo+fsOZ16RZSn6hWAayZ0L/U6h5nmABeCLNJ\nm0F5Pf+88fPll82NQ6C1JvFEJqsOpOLl4cadPVrh7eHOnI1/0rJ+Lcb1b0tM2waEt6iHZ4keP2WR\nh72EkGRQPlu2wFdfGc8WtGhhdjQua9nu4yzZ+RdrElP5q6iuP6ZtA+7s0Qp3N8WaCbF4e8gDXkJc\nifImAzelVHFjs3sp60qur3mefhoCAozhJ4RdlfXAV0LSGbq0qEfCH2d4fGBblFIs2fkXv+49TnSb\nBsS0aUDvtg1sxveXRCDElStvMlhTyroNFT2YUuoz4AbghNa6U9G6ycADwMmizZ7VWv9c0X1XmV9+\nMbqRvvsu1KtndjQ1TskHvk5mWZiwYDvzE4x+CQUWjZe7G7dENad5gC+ThnXgrZvDcJMpHoWwu/Ik\ngxfteLxZwIfAFxesf1dr/bYdj2MfhYVG1VBICIwda3Y0NVL3kECm3NqFcV9vIdS/kN+P/0mLgFoM\n7tiYmHYN6RZc3/rwVx0fT5OjFaLmKs8TyHZLBlrrlUqpYHvtr8rNng3btsGcOeAtc83a2/pDp3hx\n0W5i2zfkzu4tef/XRO7rHczzN3S8/JeFEHZlt0dolVIDlVLbr/Dr45RS25VSnymlAuwVU6VkZ8Nz\nz0FUFNx6q9nR1CjJZ7L4x+zN3DZjPWez8/Fyd+OrDUcYFurJd1tSWHsw1ewQhXA5SuuyxqCr4I6U\nugn4Vmt9yVa8ojuDn0q0GQQBqRiD4b0MNNFa31vGdx8EHgQICgrqOnfuXLvEXpoWc+YQOmMGW6dM\nIS0iokqOkZmZib+/f5Xs21mtTSngvztzUcDQ1p6E1HXjP9tzeSTchxbe2fyZW4tpW3N4JNyHqwNd\nt0HYFf82LkXKw1ZlyiM2NjZBax110Qdaa7u8gJuAwnJsFwzsrOhnF766du2qq0xqqtZ162o9dGjV\nHUNrvWLFiirdv7OwWCw6Mydfa6313mNn9WNzNuujZ7K01lp/HJ+o1ySe1FqfL481iSf1x/GJpsTq\nLFzlb6O8pDxsVaY8gE26lHOq6c8ZKKWaaK2PFS2OBHaaGQ8Ar74KGRnwxhtmR1Lt7UhOZ/KiXTSt\nV4sPbo+gfePaTL3t/J2WPPAlhHNwaDJQSs0B+gENlFLJwCSgn1IqHKOaKAl4yJExXeTwYfjwQ7jn\nHujUydRQqrMTGTm8vXQf8xKSCfTz4u9R8rCeEM6sPDOdtS7nvhpfbgOt9e2lrJ5Zzv07xsSJ4OEB\nL9qzR61rWbn/JI/M3kxuQSEPxLTmn/3bUFu6hQrh1MpzZ5BI2TOdlaTKuZ3z2rTJ6EY6cSI0a2Z2\nNNWK1pqM3ALq+HjSoWkd+rZvyJPXtqN1Q2n0E6I6KE8yuKfKo3AGWsP48dCggfGgmSi3xBMZvPTT\nHjJy8lk4NpoG/t58NCrS7LCEEBVQnofOPndEIKZbvBji4+GDD6BOHbOjqRbSs/KZGrefL9b9ga+X\nO48PbIdFg0wHLET1U64GZKVUbSAa8ATitdaZSqn2wGQgDDgBfKC1XlhVgVap4mEn2rSBBx80O5pq\nYefRdO6auYH07Hxu69aSJ69tZ50sRghR/ZSnAbkdsBxohtEu8JdS6kZgcdHyIaAzME8pNVhrvbwK\n460an38Ou3bB/Png5WV2NE4tPSufur6etGnkT992DXmgT2s6Nq1rdlhCiEoqz3AULwM5wCCgB7Ab\n+B7YArTQWncHWgK/AdVvjOesLGPimh494G9/Mzsap/Xn6SwemZ3A9e+vIie/EB9Pd6beFiGJQIga\nojzVRL2ACVrrOACl1D+BXRG3LdQAACAASURBVMAjWuscAK11llLqA+DjKou0qkydCikp8M03cInZ\nsFxV8XSSn6w8hJuCR/q1MTskIUQVKE8yaAwcLLFc/D7lgu2OAQ3tEZTDnDxpPGU8YgT07m12NE7n\naFo2N01by19ncxge3pSnh1xF03q1zA5LCFEFypMM3IDCEsvF7y98pqD6PWPw8stGNdHrr5sdiVM5\nfS6P+n5eNK3rw4CrGzEyohlRwfXNDksIUYXKOxxFsxJPIruXWJdWYpvm9gvLARIT4eOP4YEH4Kqr\nzI7G4UqbbvLnHSl8HH+IpNRzxP2rL41q+/DqyM4mRimEcJTyJoP5paz7/oLl6vUE8rPPGhPWTJpk\ndiSmKDndZNdWAUz+cRdzNv6JuxvcH9OaWp6uO3y0EK7INZ9A3rAB5s2DyZOh8WWHVKqRokMb8OGo\nCP4xezNaQ1p2PpEtA3jn1i6ENPAzOzwhhIO53hPIa9fCHXdAQAA8+aTZ0ZhGa010aAPu6tGK939N\nZHh4U967rWom8RFCOD+7TXtZLaxbB7GxkJQEmZmwY4fZEZniwPEMhn24htnr/+CrDUd4tH8bVh1I\nlekmhXBhrpUM4uMhP994b7EYyy5Ea82cjUe48cPV/HHqHG8s2cuHoyL4v0Ht+XBUBOO+3iIJQQgX\nZfpMZw7Vrx/4+EBenjHsRL9+ZkfkMOnZ+Ty7cAf/23GM3m0aEN6yHtGhgdbeRMVtCNuT02WWMSFc\nkGslg549IS7OuCPo189YdhHf/H6Epbv+4ukhV/FQn9a4uV38tLVMNymE63KtZABGAnCRJGCxaJLP\nZNMy0Jd7e4XQu01DOjSV4bmFEBdzrTYDF3IiI4e7P9vITdPXkp6dj4e7myQCIUSZXO/OwAXE7zvB\nk99u41xeAZNv7EgdH/lnFkJcmpwlapCCQgv/XrqPGSsP0T6oNnNH9aBtUG2zwxJCVAOSDGoQdzfF\nwROZ3NmjJc8N7YCPDCkhhCgnSQY1wA9bjxLZMoAW9X2ZfldXPN2lKUgIUTFy1qjGzuUW8K9523hs\n7lZmrj4MIIlACHFF5M6gmtqVks4/52zhcOo5Hu3fhkcHtDU7JCFENSbJoBpafSCVe2f9ToCfJ7Pv\n7y4PigkhKk2SQTUU0bIef7+mBU9c2476fl5mhyOEqAGkgrmaWH/oFHfN3EB2XiF+3h68PKKTJAIh\nhN04NBkopT5TSp1QSu0ssa6+UmqZUupA0c8AR8bk7AoKLUxZtp9R/1lP8plsTmTkmB2SEKIGcvSd\nwSxgyAXrJgBxWuu2QFzRsgBS0rIZ9Z8NvB93gBERzVj0z960CpRZyIQQ9ufQNgOt9UqlVPAFq4cD\n/Yrefw7EA087LCgn9vSC7exKSefdv3dhZERzs8MRQtRgSmvHzmFflAx+0lp3KlpO01rXK3qvgDPF\ny6V890HgQYCgoKCuc+fOdUjMVSUzMxN/f3+bdXmFmkINtTwUJ7IsWDQ09nONpp3SysNVSVnYkvKw\nVZnyiI2NTdBaR1243ql6E2mttVKqzOyktZ4BzACIiorS/ar55DTx8fGU/B0ST2Qw7usttG7ox7Q7\nupoXmEkuLA9XJmVhS8rDVlWUhzNcch5XSjUBKPp5wuR4HE5rzTe/H+HGD9ZwIiOXW7q2MDskIYSL\ncYZk8CMwuuj9aOAHE2OpctN/O2gzz/DZnHxu/896nl6wg4iW9Vj8WAyxVzUyMUIhhCtydNfSOcA6\noL1SKlkpdR/wBnCtUuoAMLBoucYKa17XZuL51ftPsuHwaf4e1YIv7+tOUB0fkyMUQrgiR/cmur2M\njwY4Mg4zFU88/+AXCcQ2gzXHdzFzdBT9rwoyOzQhhAtzhmoil1PHx5NzuQUsOlTAnd1bSiIQQphO\nkoGD5RVYGPtVAii4LtiDrzYcsWlDEEIIM0gycLCnF2znzzPZPHltO/5+lTcfjoqwaUMQQggzSDJw\noJ1H0/lh61Fi2jRgXH9j/oHiNoTtyekmRyeEcGVO9dBZTZdbUEh4i3p8MCrCZn10aAOZk0AIYSpJ\nBg7UtVV9FoyNxhh1QwghnIdUEznArpR03lyyl5z8QkkEQginJMmgiuUVWPjXvO3MT0gmJ7/Q7HCE\nEKJUUk1UxabFJ7Ln2Fn+c3cU9XxlZjIhhHOSO4MqtCslnQ9/TWREeFOu7SAPlgkhnJckgyqitebZ\n73ZSz9eLycM6mh2OEEJcklQTVRGlFP++KYxT53KlekgI4fQkGVSBc7kF+Hl70L5xbaC22eEIIcRl\nSTWRneUXWrhl+jpeWrTb7FCEEKLcJBnY2bQVB9l97CzdW9c3OxQhhCg3SQZ2tDvlLB/8eoDh4U0Z\n3LGx2eEIIUS5STKwk/xCC+PnbzN6D90ovYeEENWLJAM7OXA8kz9PZ/HqyE4E+EnvISFE9SK9ieyk\nQ9M6rHqqP3V9Pc0ORQghKkzuDCopv9DC91uOYrFoSQRCiGpLkkElTY8/yOPfbGX9oVNmhyKEEFdM\nkkEl7Dl2lvd/PcCNXZoS3UYmpxFCVF+SDK5Qce+hurU8eVHGHhJCVHPSgHyFPvntIDuPnmX6nZHU\nl95DNVJ+fj7Jycnk5OQ4/Nh169Zlz549Dj+us5LysFWe8vDx8aF58+Z4epavLVOSwRWKbBnAAzEh\nDOnUxOxQRBVJTk6mdu3aBAcHO3yGuoyMDGrXlnGtikl52LpceWitOXXqFMnJyYSEhJRrn5IMrlB0\nmwbSTlDD5eTkmJIIhKgspRSBgYGcPHmy3N+RNoMKmhafyOuL91Bo0WaHIhxAEoGorir6tyvJoAL2\n/ZXBu8v2k3wmG3c3OUkIIWoOp0kGSqkkpdQOpdRWpdQms+O5UH6hhX/N20YdH09ekt5D4gLTfzvI\n2oOpNuvWHkxl+m8HK7Vfd3d3wsPD6dSpE7fccgtZWVkV3sfUqVPL/F58fDw33HDDRev79etHVFSU\ndXnTpk3069ev1H0cO3bson08/vjjNGvWDIvFUmZcwcHBpKamlvl5VXr44YdZs2aNKce2lx07djBm\nzBi77c9pkkGRWK11uNY66vKbOtaMlYfYcTSdV0Z0ItDf2+xwhJMJa16XcV9vsSaEtQdTGff1FsKa\n163UfmvVqsXWrVvZuXMnXl5eTJ8+vcL7uFQyuJQTJ06wePHiy243ZcoUHnjgAeuyxWLhu+++o0WL\nFvz2228VPq4jrF+/nh49eth9vwUFBXbfZ1k6d+5McnIyR44cscv+nC0ZOKX07Hw+jj/I0LAmXNdZ\neg+5qr9/su6i15frkgCIaBFAo9re3D1zI73eiOPumRtpVNubo2eyATh9Lu+i71ZUTEwMiYmJgHEC\n7tSpE506dWLq1KkAnDt3jqFDh9KlSxc6derEN998w/vvv09KSgqxsbHExsZW6Hjjx4/n1Vdfvex2\nCxYsYMiQIdbl+Ph4OnbsyNixY5kzZ451/alTpxg0aBAdO3bk/vvvR+vz7W4jRoyga9eudOzYkRkz\nZljX+/v7M378eLp168bAgQPZuHEj/fr1o3Xr1vz4448XxfKPf/zDun7kyJHce++9AHz22WdMnDgR\ngD179tCuXTvc3d2t3yssLCQkJAStNWlpabi7u7Ny5UoA+vTpw4EDB9i4cSM9e/YkIiKC6Oho9u3b\nB8CsWbMYNmwY/fv3Z8CAAcTHx9O3b1+GDx9O69atmTBhArNnz6Zbt2507tyZgwdLv1v09/e3vp8/\nf771qn/MmDE8/PDDREVF0a5dO3766SfrdjfeeCNz58691D9PuTlTbyIN/KKU0sAnWusZF26glHoQ\neBAgKCiI+Ph4hwU3IcqTOt7pdj1mZmamQ38HZ+ds5VG3bl0yMjKsy4WFhRdtk5OTS0ZGBtn5hfh7\nudHQ34ujaTk0qeONv5cbOTk5ZGRkkJmVf9H3S+77QoWFhdbPMzIyKCgoYNGiRQwcOJCVK1cyc+ZM\n4uLi0FrTv39/oqKiSEpKomHDhtaTQ3p6OnXr1uWdd95h0aJFBAYGXnTMrKwsCgoKLlpfWFhIWFgY\n8+fP53//+x/+/v42MRVLSkqibt265OXlkZeXB8AXX3zBiBEjGDhwIM888wynT5/G09OTiRMncs01\n17BgwQKWLFnCzJkzyczMxNvbm/fee4/69euTnZ1Nv379GDRoEIGBgZw7d44ePXowceJE7rrrLiZM\nmMDChQvZu3cvDz/88EUJLioqiri4OGJjYzly5AjJyclkZGTw66+/ctNNN5GRkcH3339Pv379Lvpd\nQkND+f3330lKSqJLly4sX76cDh06cOTIERo3boyvry8///wzHh4erFixgqeeeoqvvvqKnJwcEhIS\nWLt2LfXr12fVqlVs27aN33//nYCAAMLCwrj77ruJi4tj2rRpvPPOO7z55pul/rsXx5SdnU1+fj4Z\nGRnW513i4uI4dOgQN9xwAwkJCQB06NCBKVOmMHbs2FL3l5OTU/7/U1prp3gBzYp+NgK2AX0utX3X\nrl21Ixw+mVll+16xYkWV7bs6crby2L17d4W2X5N4Uke89It+Z+leHfHSL3pN4skrPvbZs2e11lq7\nubnpLl266C5duuhx48bp3NxcPXXqVP38889bt33uuef0e++9p/ft26dbtWqln3rqKb1y5Urr561a\ntdInT5Yey4oVK/TQoUMvWt+3b1/9+++/67i4ON2/f3/9+++/6759+178O69ZowcPHmxdzs3N1U2b\nNrXGP3LkSL1o0SKttdZdunTRBw8etG4bEBBgjWvSpEk6LCxMh4WF6Tp16uh169ZprbX28vLSFotF\nnz17Vj///PP6lVde0VprXVhYqOvWrXtRPMnJybp79+56165devTo0XrYsGE6JSVFt2/f3hrToEGD\n9NGjRy/67iuvvKKnTZumx48frxcsWKCHDBmiV61apW+55RattdZHjhzRI0aM0B07dtSdOnXS7du3\n11pr/d///lePGTPGpkwHDhxoXY6JidGrV6/WWmsdFxenhw8fftGxtdbaz8/P+n7evHl69OjRWmut\nR48erWfOnFnq/vbv368jIyNL3Z/Wpf8NA5t0KedUp6km0lofLfp5AvgO6GZuREbvoUHvruSLdUlm\nhyKcXHEbwYejIvi/Qe35cFSETRvClSpuM9i6dSsffPABXl5lP+3erl07Nm/eTOfOnXnuued46aWX\nLtrmu+++Izw8nPDwcDZtunw/jf79+5Odnc369evLjK/kE9pLly4lLS2Nzp07ExwczOrVq22qikoT\nHx/P8uXLWbduHdu2bSMiIsK6T09PT2sXSTc3N7y9va3vS6ufb9asGWlpaSxZsoQ+ffoQExPDt99+\ni7+/P7Vr1yYrK4u0tDSaNm160Xf79OnDqlWr2LhxI9dffz1paWnEx8cTExMDwPPPP09sbCw7d+5k\n0aJFNr+3n5+fzb6K4ywr7sLCQuu/wwsvvADYdgW98Kn3C7uJFi/n5ORQq1atMsu2IpwiGSil/JRS\ntYvfA4OAnWbGVFA09pC/jwdDpZ1AXMb25HQ+HBVBdKjxIGJ0aAM+HBXB9uR0ux8rJiaG77//nqys\nLM6dO8d3331HTEwMKSkp+Pr6cueddzJ+/Hg2b94MQO3ata3VDyNHjrQml5K9hS7lueee49///nep\nn7Vr146kpCTr8pw5c/j0009JSkoiKSmJw4cPs2zZMrKysujTpw9ff/01AIsXL+bMmTOAUZ0VEBCA\nr68ve/fuLTPxlFePHj2YOnWqNRm8/fbb1hP6ihUrymw76datG2vXrsXNzQ0fHx/Cw8P55JNP6NOn\njzXOZs2aAUY7QWW4u7tb/x2Kk3ZQUBB79uyxNsCXNG/ePCwWCwcPHuTQoUO0bdsWgP3799OpU6dK\nxVLMKZIBEASsVkptAzYC/9NaLzEzoE9WHmJ7cjovD5feQ+LyHu4bak0ExaJDG/Bw31C7HysyMpIx\nY8bQrVs3unfvzv33309ERAQ7duygW7duhIeH8+KLL/Lcc88B8OCDDzJkyJAyT4JxcXE0b97c+lq3\nzrZx+/rrr6dhw4alftfPz4/Q0FASExPJyspiyZIlDB061Obz3r17s2jRIiZNmsTKlSvp2LEjCxcu\npGXLlgAMGTKEgoICrr76aiZMmFDpXj4xMTEUFBTQpk0bIiMjOX36tDUZLF682KaxuyRvb29atGhh\nPX5MTAwZGRl07twZgKeeeopnnnmGiIiIKuk19MYbb3DDDTcQHR1Nkya2F6AtW7akW7duXHfddUyf\nPh0fHx/ASG4ly7tSSqs7qg6vqmwz2PfXWd322Z/12K82VdkxtHa+OnKzOVt5VLTNwJ6K67erg4UL\nF+qJEydW6THsVR4RERE6Ly/PLvtylNGjR+t58+bZrDt79qzOycnR3bt31/n5+WV+tyJtBs7Um8hp\nHD2TTfOAWrw03D63X0LUZCNHjuTUqeoxuVNx1VlNcOTIEd544w08POxzGpdkUIrYqxrRp11DGXJC\niHK6//77zQ6hxiqrfaJt27bWtgN7cJY2A6ew/3gGn69NwmLRkgiEEC5F7gyKFBSNPZR8JpsbuzSV\nCWuEEC5FkkGRGasOWbsHSiIQQrgaqSYCDhzPYOqyA1zXqbE8UyCEcEkunwy01oyfvx0/b3deGt5J\nJjMRTsUeQ1gXS0pKKvUBpTFjxtCsWTNyc3MBSE1NJTg4uNR9ZGdn07dvX5txlqZOnYqPjw/p6WU/\nYNevX79yPfFcFd544w1mz55tyrGr2smTJ8t8bqKiXD4ZKKV4anB73rq5Cw1ry8NlopLWrYPXXzd+\n2sHlhrDWWl9yzoDycnd357PPPrvsdp999hl/+9vfbEb8nDNnDtdccw0LFy6sdBxVYenSpQwaNMhh\nx3PkMNYNGzakSZMmdpmbwaWTQUGh8Z8ouk0DBnYIMjka4dQefxz69bv0KyICeveGZ581fkZEXHr7\nxx+vUAjFQ1gnJSXRvn177r77bjp16sSff/7JL7/8Qs+ePYmMjOSWW24hMzOzgr/e47z77ruXPZHN\nnj2b4cOHW5cPHjxIZmYmr7zyis0YRNnZ2dx2221cffXVjBw5kuzsbOtnY8eOJSoqio4dOzJp0iTr\n+uDgYJ555hnCw8OJiopi8+bNDB48mNDQUGbOnHlRLG+99Rbvv/8+AE888QT9+/cH4Ndff+WOO+4A\n4OzZs+Tl5dGwYUOSkpLo378/YWFhDBgwoNR5ADp37kxaWhpaawIDA/niiy8AuPvuu1m2bBlJSUnE\nxMQQGRlJZGQka9euBbCOYTRs2DA6dOhAUlISV111FWPGjKFdu3bccccdLF++nF69etG2bVs2btxY\navmWnPCn5IRCkydP5q677qJnz560bdvWprvpiBEj7HLn47LJoNCi+fuM9Xy0ItHsUERNkZ4OxVfp\nFouxbCcFBQUsXrzYOjTCgQMHeOSRR9i1axd+fn688sorLF++nM2bNxMVFcWUKVMqtP+WLVvSu3dv\nvvzyyzK3ycvL49ChQzZVSHPnzuW2224jJiaGffv2cfz4cQA+/vhjfH192bNnDy+++KJ1yGWAV199\nlU2bNrF9+3Z+++03tm/fbhPH1q1biYmJYcyYMcyfP5/169fz2muvXRRPTEwMq1atAowTZ2ZmJvn5\n+axatco6ntDy5csZMGAAAP/85z8ZPXo027dv54477uDRRx+9aJ+9evVizZo17Nq1i9atW1v3v27d\nOqKjo2nUqBHLli1j8+bNfPPNNzb72Lx5M++99x779+8HIDExkSeffJK9e/eyd+9evv76a1avXs3b\nb79d6u9zOdu3b+fXX39l3bp1vPnmm6SkpADGsN3FcVaGy/Ym+s+qQyT8cYYx0cFmhyKqg6IJZC5p\n3ToYMADy8sDLC2bPhp49K3XY7OxswsPDAePkd99995GSkkKrVq2sY+isX7+e3bt306tXL8A4afe8\nguM+88wzDB8+vMyxblJTU6lXr57Nujlz5vDdd9/h5ubGTTfdxLx58xg3bhwrV660nijDwsIICwuz\nfufbb79lxowZFBQUcOzYMXbv3m39fNiwYYBxhZ6ZmUnt2rWpXbs23t7epKWl2Ry/a9euJCQkcPbs\nWby9vYmMjGTTpk2sWrXKesewZMkS7rnnHsA4oRdXZd1111089dRTF/2OMTExrFy5klatWjF27Fhm\nzJjB0aNHCQgIwM/Pj/T0dMaNG8fWrVtxd3e3nvjBGOguJCTEuhwSEmJN3h07dmTAgAEopejcubPN\n4H7lNXz4cGrVqkWtWrWIiYlh48aNjBgxgkaNGlkTQ2W4TDKY/ttBwprXJTq0AYknMpiybD/XBAeQ\nfObKG+SEsNGzJ8TFQXy8UQ1UyUQA59sMLlRyyGStNddee+1FQ0Vv2LCBhx56CICXXnrJ5oRcmrZt\n2xIeHs63335bZiwlh1besWMHBw4c4NprrwWMJBQSEsK4cePKPMbhw4d5++23rRO/jBkzxmafJYd6\nvnAY6AursDw9PQkJCWHWrFlER0cTFhbGihUrSExM5OqrrwZg48aNfPzxx5f8vUvq06cPH330EUeO\nHOHVV1/lu+++Y/78+daB7t59912CgoLYtm0bFovFOmAcVHwYa4DBgwdz/PhxoqKi+PTTT/Hw8LC2\nATl6GGuXqSYqnqN29YFU/jVvO17uigMnMunSot7lvyxEefXsCc88Y5dEUF49evRgzZo11ikxz507\nx/79++nevbt1mOTiK+7LmThxIm+//XapnwUEBFBYWGg9Sc2ZM4fJkydbh6tOSUkhJSWFP/74w2a4\n6p07d1qrgs6ePYufnx9169bl+PHj5Zpj+VKKh6guHq56+vTpREREoJRi165dXHXVVdbG7ujoaOss\ncLNnz7ae4Etq0aIFqampHDhwgNatW9O7d2/r/sEYxrpJkya4ubnx5Zdfljr7XUUsXbqUrVu38umn\nnwJGm0FxldqCBQtstv3hhx/Iycnh1KlTrF69mmuuuQaw3zDWLpMMiseXf2R2AtuT09Aopt0RedGw\nw0JUNw0bNmTWrFncfvvthIWF0bNnT/bu3Vvqtvv27bMZrnrevHk2n3fs2JHIyMgyjzVo0CBWr14N\nGO0FI0eOtPl85MiRzJ07l7Fjx5KZmcnVV1/NCy+8QNeuXQHo0qULERERXHXVVYwaNcpatXWlYmJi\nOHbsGD179iQoKAgfH58yh6v+4IMP+O9//0tYWBhffvkl7733Xqn77N69O+3atbPu/+jRo/Tu3RuA\nRx55hM8//5wuXbqwd+/ei+4GKmvSpEk89thjREVF2fTYAqO6LTY2lh49evDUU09ZJ+ix2zDWpQ1l\nWh1eVzqE9TtL9+pWT/+k316694q+b0/ONmSz2ZytPGQI64slJCToO++80+HHvZLyGDhwoE5JSamC\naBxv0qRJ+q233rIulyyPmJgYffr06VK/Vy2nvXSEtQdT+WrDER7t34bZG45UekpCIVxNZGQksbGx\nla4ecYRly5ZdNElMTXPy5En+7//+j4CAgErvy2UakEvOURsd2oAeoYE2y0KI8rn33nvNDsHlTJ48\nudT1DRs2ZMSIEXY5hsvcGThyjlpRcxh31UJUPxX923WZO4PS5qKNDm0gdwWiTD4+Ppw6dYrAwEAZ\ns0pUK1prTp06ZdP19XJcJhkIUVHNmzcnOTmZkydPOvzYOTk5FfqPXNNJedgqT3n4+PjQvHnzcu9T\nkoEQZSh+qMkM8fHxREREmHJsZyTlYasqysNl2gyEEEKUTZKBEEIISQZCCCFAVdeuc0qpk8AfZsdR\nSQ0AefLtPCmP86QsbEl52KpMebTSWje8cGW1TQY1gVJqk9Y6yuw4nIWUx3lSFrakPGxVRXlINZEQ\nQghJBkIIISQZmG2G2QE4GSmP86QsbEl52LJ7eUibgRBCCLkzEEIIIclACCEEkgxMoZRqoZRaoZTa\nrZTapZR6zOyYzKaUcldKbVFK/WR2LGZTStVTSs1XSu1VSu1RSjluQmUno5R6ouj/yE6l1ByllEuN\nVqeU+kwpdUIptbPEuvpKqWVKqQNFPys/sw2SDMxSADypte4A9AD+oZTqYHJMZnsM2GN2EE7iPWCJ\n1voqoAsuWi5KqWbAo0CU1roT4A7cZm5UDjcLGHLBuglAnNa6LRBXtFxpkgxMoLU+prXeXPQ+A+M/\nezNzozKPUqo5MBT41OxYzKaUqgv0AWYCaK3ztNZp5kZlKg+gllLKA/AFUkyOx6G01iuB0xesHg58\nXvT+c8AuU51JMjCZUioYiAA2mBuJqaYCTwEWswNxAiHASeC/RdVmnyql/MwOygxa66PA28AR4BiQ\nrrX+xdyonEKQ1vpY0fu/gCB77FSSgYmUUv7AAuBxrfVZs+Mxg1LqBuCE1jrB7FichAcQCXystY4A\nzmGnaoDqpqgufDhGgmwK+Cml7jQ3KueijWcD7PJ8gCQDkyilPDESwWyt9UKz4zFRL2CYUioJmAv0\nV0p9ZW5IpkoGkrXWxXeK8zGSgysaCBzWWp/UWucDC4Fok2NyBseVUk0Ain6esMdOJRmYQBkT6s4E\n9mitp5gdj5m01s9orZtrrYMxGgd/1Vq77NWf1vov4E+lVPuiVQOA3SaGZKYjQA+llG/R/5kBuGhj\n+gV+BEYXvR8N/GCPnUoyMEcv4C6Mq+CtRa/rzQ5KOI1/ArOVUtuBcOA1k+MxRdHd0XxgM7AD43zl\nUsNSKKXmAOuA9kqpZKXUfcAbwLVKqQMYd09v2OVYMhyFEEIIuTMQQgghyUAIIYQkAyGEEEgyEEII\ngSQDIYQQSDIQLkIpNUYppct4mTb2j1JqllIq2azjC1HMw+wAhHCwWzCe8i2pwIxAhHAmkgyEq9mq\ntU40OwghnI1UEwlRpERVUh+l1PdKqUyl1Cml1EdKqVoXbNtEKfWFUipVKZWrlNpe2iBqSqkQpdSX\nSqm/irY7pJR6r5TtIpRSq5RSWUWTljx8weeNlVKfK6VSivZzTCn1k1Kqkf1LQrgiuTMQrsa9aGz8\nkixa65LDZ38FfAtMNZQUpwAAAohJREFUA7oBLwB+wBiAoiGlfwMCgGeBP4E7gS+VUr5a6xlF24UA\nG4Gson0cAFoCgy44fh3ga4yhvF8C7gE+Vkrt01qvKNrmS6AVML7oeEEYY/X4XmlBCGFDay0vedX4\nF8aJXJfx+umCbaZf8N2JQCHQrmh5XNF2/S7YbjnGCJLuRctfAJlA00vENatoX7El1nkDp4AZJdZl\nAo+aXY7yqrkvuTMQrmYkFzcgX9ib6NsLlucCr2DcJezHmInsqNY6/oLtvgL+C3TAGFhtEEaiudzs\nXFn6/B0AWutcpdR+jLuIYr8D44tG7/wV2Km1loHFhN1IMhCuZqe+fAPy8TKWi6cmrY8x89aF/irx\nOUAgFyee0pwpZV0uUHLy978DkzBmhJsKHFNKTQde0bZVXEJcEWlAFuJiF04jWLx8tOjnaaBxKd9r\nXOJzgFTsNLe11vqE1vofWutmwFUY1UsvAg/ZY/9CSDIQ4mK3XrB8G8b8zMWzj/0GNFdK9bpgu1EY\nbQbFk9H8AtxQPCuVvWit92mtn8W4o+hkz30L1yXVRMLVhCulGpSyflOJ99crpd7COJl3w6ie+UJr\nfaDo81nAY8BCpdREjKqgO4BrgYe01oVF200CrgfWKqVeAxIx7hSG6ArM5qaUqovROD0b2AvkY8wN\nHFAUoxCVJslAuJp5ZaxvWOL9ncCTwFggD/gP8K/iD7XW55RSfYF/Y8wyVRvYB9yltf6qxHZJSqke\nGI3PrwP+GFVNFZ2mMAdjtq8HMLqXWoqOd4fW2i5THgohM50JUUQpNQajN1DbcjQyC1GjSJuBEEII\nSQZCCCGkmkgIIQRyZyCEEAJJBkIIIZBkIIQQAkkGQgghkGQghBAC+H+dGHe+xtzcJwAAAABJRU5E\nrkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IyJexWpK6EH8",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68
        },
        "outputId": "b64d0463-666c-4e78-c305-0ee8296fcb0b"
      },
      "source": [
        "source_sentence = [\"<sos>\"] + train_data[5].src + [\"<eos>\"]\n",
        "target_sentence = [\"<sos>\"] + train_data[5].trg + [\"<eos>\"]\n",
        "pred_token = translate_sentence(model, source_sentence, GER, ENG)\n",
        "print('source:', ' '.join(source_sentence))\n",
        "print('target:', ' '.join(target_sentence))\n",
        "print('inference:', ' '.join(pred_token))"
      ],
      "execution_count": 39,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "source: <sos> ein mann in grün hält eine gitarre , während der andere mann sein hemd ansieht . <eos>\n",
            "target: <sos> a man in green holds a guitar while the other man observes his shirt . <eos>\n",
            "inference: a man in green is holding a green guitar while the other man is playing his guitar .\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "y_29xud9AbFo",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}