train_rnn_on_wikidata.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "0. Import stuff\n",
    "\n",
    "1. Load model\n",
    "    - a) The RNN class and functions it uses\n",
    "    - b) Load the model\n",
    " \n",
    "2. Load the data set\n",
    "    - that has been saved \n",
    "    - a) training set\n",
    "    - b) test set\n",
    "        - The test set should have the same amount of KG context as in the training set\n",
    " \n",
    "3. Training\n",
    "    - a) Training function\n",
    "    - b) Run the training\n",
    "    - c) Plot the losses from training \n",
    "    - d) Save the trained model\n",
    "\n",
    "4. Testing\n",
    "    - a) Test function\n",
    "    - b) Call the function and display / save results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import random\n",
    "import requests\n",
    "import numpy as np\n",
    "\n",
    "from gensim.models import KeyedVectors\n",
    "import logging\n",
    "\n",
    "from gensim import utils\n",
    "import nltk\n",
    "\n",
    "import networkx as nx\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import shutil\n",
    "import smart_open\n",
    "import sys \n",
    "\n",
    "from sys import platform\n",
    "import random\n",
    "import tensorflow as tf\n",
    "import json\n",
    "\n",
    "import time\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "TINY = 1e-6\n",
    "ONE = tf.constant(1.)\n",
    "NAMESPACE = 'rnn'\n",
    "forbidden_weight = 1.\n",
    "_weight_for_positive_matches = 1.\n",
    "_rw = 1e-1\n",
    "\n",
    "\n",
    "class RNN(object):\n",
    "    _nodes_vocab_size = 300 * 3\n",
    "    _question_vocab_size = 300\n",
    "    _question_vector_size = 150\n",
    "    _types_size = 3\n",
    "    _mask_size = 200\n",
    "    _types_proj_size = 5\n",
    "    _word_proj_size = 50\n",
    "    _word_proj_size_for_rnn = 50\n",
    "    _word_proj_size_for_item = 50\n",
    "    _internal_proj_size = 250\n",
    "    _hidden_layer1_size = 250\n",
    "    _hidden_layer2_size = 250\n",
    "    _output_size = 2\n",
    "\n",
    "    _memory_dim = 100\n",
    "    _stack_dimension = 2\n",
    "\n",
    "    def __init__(self, dropout=1.0):\n",
    "        tf.compat.v1.reset_default_graph()\n",
    "        with tf.compat.v1.variable_scope(NAMESPACE):\n",
    "            config = tf.compat.v1.ConfigProto(allow_soft_placement=True)\n",
    "            self.sess = tf.compat.v1.Session(config=config)\n",
    "\n",
    "            # Input variables\n",
    "            self.node_X_fw = tf.compat.v1.placeholder(tf.float32, shape=(None, None, self._nodes_vocab_size), name='node_X_fw')\n",
    "            self.node_X_bw = tf.compat.v1.placeholder(tf.float32, shape=(None, None, self._nodes_vocab_size), name='node_X_bw')\n",
    "            self.question_vectors_fw = tf.compat.v1.placeholder(tf.float32, shape=(None, None, self._question_vocab_size),\n",
    "                                                      name='question_vectors_inp_fw')\n",
    "            self.question_vectors_bw = tf.compat.v1.placeholder(tf.float32, shape=(None, None, self._question_vocab_size),\n",
    "                                                      name='question_vectors_inp_nw')\n",
    "            self.question_mask = tf.compat.v1.placeholder(tf.float32, shape=(None, None, self._mask_size),\n",
    "                                                name='question_mask')\n",
    "\n",
    "            # The question is pre-processed by a bi-GRU\n",
    "            self.Wq = tf.Variable(tf.compat.v1.random_uniform([self._question_vocab_size,\n",
    "                                                     self._word_proj_size_for_rnn], -_rw, _rw))\n",
    "            self.bq = tf.Variable(tf.compat.v1.random_uniform([self._word_proj_size_for_rnn], -_rw, _rw))\n",
    "            self.internal_projection = lambda x: tf.nn.relu(tf.matmul(x, self.Wq) + self.bq)\n",
    "            self.question_int_fw = tf.map_fn(self.internal_projection, self.question_vectors_fw)\n",
    "            self.question_int_bw = tf.map_fn(self.internal_projection, self.question_vectors_bw)\n",
    "\n",
    "            self.rnn_cell_fw = tf.compat.v1.nn.rnn_cell.MultiRNNCell([tf.compat.v1.nn.rnn_cell.GRUCell(self._memory_dim) for _ in range(self._stack_dimension)],\n",
    "                                                state_is_tuple=True)\n",
    "            self.rnn_cell_bw = tf.compat.v1.nn.rnn_cell.MultiRNNCell([tf.compat.v1.nn.rnn_cell.GRUCell(self._memory_dim) for _ in range(self._stack_dimension)],\n",
    "                                                state_is_tuple=True)\n",
    "            with tf.compat.v1.variable_scope('fw'):\n",
    "                output_fw, state_fw = tf.compat.v1.nn.dynamic_rnn(self.rnn_cell_fw, self.question_int_fw, time_major=True,\n",
    "                                                        dtype=tf.float32)\n",
    "            with tf.compat.v1.variable_scope('bw'):\n",
    "                output_bw, state_bw = tf.compat.v1.nn.dynamic_rnn(self.rnn_cell_bw, self.question_int_bw, time_major=True,\n",
    "                                                        dtype=tf.float32)\n",
    "\n",
    "            self.states = tf.concat(values=[output_fw, tf.reverse(output_bw, [0])], axis=2)\n",
    "            self.question_vector_pre = tf.reduce_mean(tf.multiply(self.question_mask, self.states), axis=0)\n",
    "            self.Wqa = tf.Variable(\n",
    "                tf.compat.v1.random_uniform([2 * self._memory_dim, self._question_vector_size], -_rw, _rw),\n",
    "                name='Wqa')\n",
    "            self.bqa = tf.Variable(tf.compat.v1.random_uniform([self._question_vector_size], -_rw, _rw), name='bqa')\n",
    "            self.question_vector = tf.nn.relu(tf.matmul(self.question_vector_pre, self.Wqa) + self.bqa)\n",
    "\n",
    "            # bi-LSTM of triplets part\n",
    "            self.Wtri = tf.Variable(tf.compat.v1.random_uniform([self._nodes_vocab_size,\n",
    "                                                       self._word_proj_size_for_rnn], -_rw, _rw))\n",
    "            self.btri = tf.Variable(tf.compat.v1.random_uniform([self._word_proj_size_for_rnn], -_rw, _rw))\n",
    "            self.internal_projection = lambda x: tf.nn.relu(tf.matmul(x, self.Wtri) + self.btri)\n",
    "            self.node_int_fw = tf.map_fn(self.internal_projection, self.node_X_fw)\n",
    "            self.node_int_bw = tf.map_fn(self.internal_projection, self.node_X_bw)\n",
    "\n",
    "            self.rnn_cell_fw_tri = tf.compat.v1.nn.rnn_cell.MultiRNNCell(\n",
    "                [tf.compat.v1.nn.rnn_cell.GRUCell(self._memory_dim) for _ in range(self._stack_dimension)],\n",
    "                state_is_tuple=True)\n",
    "            self.rnn_cell_bw_tri = tf.compat.v1.nn.rnn_cell.MultiRNNCell(\n",
    "                [tf.compat.v1.nn.rnn_cell.GRUCell(self._memory_dim) for _ in range(self._stack_dimension)],\n",
    "                state_is_tuple=True)\n",
    "            with tf.compat.v1.variable_scope('fw_tri'):\n",
    "                output_fw_tri, _ = tf.compat.v1.nn.dynamic_rnn(self.rnn_cell_fw_tri, self.node_int_fw, time_major=True,\n",
    "                                                     dtype=tf.float32)\n",
    "            with tf.compat.v1.variable_scope('bw_tri'):\n",
    "                output_bw_tri, _ = tf.compat.v1.nn.dynamic_rnn(self.rnn_cell_bw_tri, self.node_int_bw, time_major=True,\n",
    "                                                     dtype=tf.float32)\n",
    "\n",
    "            self.states_tri = tf.concat(values=[output_fw_tri, output_bw_tri], axis=2)\n",
    "\n",
    "            self.Walpha = tf.Variable(tf.compat.v1.random_uniform([2 * self._memory_dim + self._question_vector_size,\n",
    "                                                         1], -_rw, _rw))\n",
    "            self.balpha = tf.Variable(tf.compat.v1.random_uniform([1], -_rw, _rw))\n",
    "            self.alpha_projection = lambda x: tf.nn.relu(tf.matmul(tf.concat(values=[x, self.question_vector], axis=1),\n",
    "                                                                   self.Walpha) + self.balpha)\n",
    "            self.alphas = tf.map_fn(self.alpha_projection, self.states_tri)\n",
    "            self.alphas = tf.compat.v1.nn.softmax(self.alphas, dim =0)\n",
    "            self.centroid = tf.reduce_mean(tf.multiply(self.states_tri, self.alphas), axis=0)\n",
    "\n",
    "            self.concatenated = tf.concat(values=[self.question_vector, self.centroid], axis=1)\n",
    "\n",
    "            # Final feedforward layers\n",
    "            self.Ws1 = tf.Variable(\n",
    "                tf.compat.v1.random_uniform([self._question_vector_size\n",
    "                                   + 2 * self._memory_dim,\n",
    "                                   self._hidden_layer2_size], -_rw, _rw),\n",
    "                name='Ws1')\n",
    "            self.bs1 = tf.Variable(tf.compat.v1.random_uniform([self._hidden_layer2_size], -_rw, _rw), name='bs1')\n",
    "            self.first_hidden = tf.nn.relu(tf.matmul(self.concatenated, self.Ws1) + self.bs1)\n",
    "            self.first_hidden_dropout = tf.nn.dropout(self.first_hidden, dropout)\n",
    "\n",
    "            self.Wf = tf.Variable(\n",
    "                tf.compat.v1.random_uniform([self._hidden_layer2_size, self._output_size], -_rw,\n",
    "                                  _rw),\n",
    "                name='Wf')\n",
    "            self.bf = tf.Variable(tf.compat.v1.random_uniform([self._output_size], -_rw, _rw), name='bf')\n",
    "            self.outputs = tf.nn.softmax(tf.matmul(self.first_hidden_dropout, self.Wf) + self.bf)\n",
    "\n",
    "            # Loss function and training\n",
    "            self.y_ = tf.compat.v1.placeholder(tf.float32, shape=(None, self._output_size), name='y_')\n",
    "            self.outputs2 = tf.squeeze(self.outputs)\n",
    "            self.y2_ = tf.squeeze(self.y_)\n",
    "            self.one = tf.ones_like(self.outputs)\n",
    "            self.tiny = self.one * TINY\n",
    "            self.cross_entropy = (tf.reduce_mean(\n",
    "                -tf.reduce_sum(self.y_ * tf.compat.v1.log(self.outputs + self.tiny) * _weight_for_positive_matches\n",
    "                               + (self.one - self.y_) * tf.compat.v1.log(\n",
    "                    self.one - self.outputs + self.tiny))\n",
    "            ))\n",
    "\n",
    "        # Clipping the gradient\n",
    "        optimizer = tf.compat.v1.train.AdamOptimizer(1e-4)\n",
    "        gvs = optimizer.compute_gradients(self.cross_entropy)\n",
    "        capped_gvs = [(tf.clip_by_value(grad, -1., 1.), var) for grad, var in gvs if var.name.find(NAMESPACE) != -1]\n",
    "        self.train_step = optimizer.apply_gradients(capped_gvs)\n",
    "        self.sess.run(tf.compat.v1.global_variables_initializer())\n",
    "\n",
    "        # Adding the summaries\n",
    "        tf.compat.v1.summary.scalar('cross_entropy', self.cross_entropy)\n",
    "        self.merged = tf.compat.v1.summary.merge_all(key=tf.compat.v1.GraphKeys.SUMMARIES)\n",
    "        self.train_writer = tf.compat.v1.summary.FileWriter('./train', self.sess.graph)\n",
    "\n",
    "    def __train(self, node_X, item_vector, question_vectors, question_mask, y):\n",
    "        node_X_fw = np.array(node_X)\n",
    "        node_X_fw = np.transpose(node_X_fw, (1, 0, 2))\n",
    "        node_X_bw = node_X_fw[::-1, :, :]\n",
    "\n",
    "        question_vectors = np.array(question_vectors)\n",
    "        question_vectors_fw = np.transpose(question_vectors, (1, 0, 2))\n",
    "        question_vectors_bw = question_vectors_fw[::-1, :, :]\n",
    "\n",
    "        question_mask = np.array(question_mask)\n",
    "        question_mask = np.transpose(question_mask, (1, 0, 2))\n",
    "\n",
    "        y = np.array(y)\n",
    "\n",
    "        feed_dict = {}\n",
    "        feed_dict.update({self.node_X_fw: node_X_fw})\n",
    "        feed_dict.update({self.node_X_bw: node_X_bw})\n",
    "        feed_dict.update({self.question_vectors_fw: question_vectors_fw})\n",
    "        feed_dict.update({self.question_vectors_bw: question_vectors_bw})\n",
    "        feed_dict.update({self.question_mask: question_mask})\n",
    "        feed_dict.update({self.y_: y})\n",
    "\n",
    "        loss, _,  outputs2, y2 = self.sess.run(\n",
    "            [self.cross_entropy, self.train_step, self.outputs2, self.y2_], feed_dict)\n",
    "        return loss\n",
    "\n",
    "    def train(self, data, epochs=20):\n",
    "        \n",
    "        for epoch in range(epochs):\n",
    "            loss = self.__train([data[i][0] for i in range(len(data))],\n",
    "                                   [data[i][1] for i in range(len(data))],\n",
    "                                   [data[i][2] for i in range(len(data))],\n",
    "                                   [data[i][3] for i in range(len(data))],\n",
    "                                   [data[i][4] for i in range(len(data))])\n",
    "\n",
    "            sys.stdout.flush()\n",
    "        return loss\n",
    "    def __predict(self, node_X, item_vector, question_vectors, question_mask):\n",
    "        \n",
    "        node_X_fw = np.array(node_X)\n",
    "        node_X_fw = np.transpose(node_X_fw, (1, 0, 2))\n",
    "        node_X_bw = node_X_fw[::-1, :, :]\n",
    "\n",
    "        question_vectors = np.array(question_vectors)\n",
    "        question_vectors_fw = np.transpose(question_vectors, (1, 0, 2))\n",
    "        question_vectors_bw = question_vectors_fw[::-1, :, :]\n",
    "\n",
    "        question_mask = np.array(question_mask)\n",
    "        question_mask = np.transpose(question_mask, (1, 0, 2))\n",
    "\n",
    "        feed_dict = {}\n",
    "        feed_dict.update({self.node_X_fw: node_X_fw})\n",
    "        feed_dict.update({self.node_X_bw: node_X_bw})\n",
    "\n",
    "        feed_dict.update({self.question_vectors_fw: question_vectors_fw})\n",
    "        feed_dict.update({self.question_vectors_bw: question_vectors_bw})\n",
    "        feed_dict.update({self.question_mask: question_mask})\n",
    "\n",
    "        y_batch = self.sess.run([self.outputs2], feed_dict)\n",
    "        return y_batch\n",
    "\n",
    "    def __standardize_item(self, item):\n",
    "        if item[0] < item[1]:\n",
    "            return [0., 1.]\n",
    "        return [1., 0.]\n",
    "\n",
    "    def predict(self, node_X, item_vector, question_vectors, question_mask):\n",
    "        output = self.__predict([node_X], [item_vector], [question_vectors], [question_mask])\n",
    "        return self.__standardize_item(output[0])\n",
    "\n",
    "    # Loading and saving functions\n",
    "\n",
    "    def save(self, filename):\n",
    "        saver =  tf.compat.v1.train.Saver()\n",
    "        saver.save(self.sess, filename)\n",
    "\n",
    "    def load_tensorflow(self, filename):\n",
    "        saver =  tf.compat.v1.train.Saver([v for v in tf.compat.v1.global_variables() if NAMESPACE in v.name])\n",
    "        saver.restore(self.sess, filename)\n",
    "\n",
    "    @classmethod\n",
    "    def load(self, filename, dropout=1.0):\n",
    "        model = GCN_QA(dropout)\n",
    "        model.load_tensorflow(filename)\n",
    "        return model\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "b) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading model: \n",
      "WARNING:tensorflow:From <ipython-input-6-7678756ad5a1>:51: GRUCell.__init__ (from tensorflow.python.keras.layers.legacy_rnn.rnn_cell_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This class is equivalent as tf.keras.layers.GRUCell, and will be replaced by that in Tensorflow 2.0.\n",
      "WARNING:tensorflow:From <ipython-input-6-7678756ad5a1>:51: MultiRNNCell.__init__ (from tensorflow.python.keras.layers.legacy_rnn.rnn_cell_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This class is equivalent as tf.keras.layers.StackedRNNCells, and will be replaced by that in Tensorflow 2.0.\n",
      "WARNING:tensorflow:From <ipython-input-6-7678756ad5a1>:56: dynamic_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `keras.layers.RNN(cell)`, which is equivalent to this API\n",
      "WARNING:tensorflow:From /Users/valdimareggertsson/.pyenv/versions/3.8.2/lib/python3.8/site-packages/tensorflow/python/keras/layers/legacy_rnn/rnn_cell_impl.py:561: Layer.add_variable (from tensorflow.python.keras.engine.base_layer_v1) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `layer.add_weight` method instead.\n",
      "WARNING:tensorflow:From /Users/valdimareggertsson/.pyenv/versions/3.8.2/lib/python3.8/site-packages/tensorflow/python/keras/layers/legacy_rnn/rnn_cell_impl.py:570: calling Constant.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
      "WARNING:tensorflow:From /Users/valdimareggertsson/.pyenv/versions/3.8.2/lib/python3.8/site-packages/tensorflow/python/keras/layers/legacy_rnn/rnn_cell_impl.py:580: calling Zeros.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
      "WARNING:tensorflow:From /Users/valdimareggertsson/.pyenv/versions/3.8.2/lib/python3.8/site-packages/tensorflow/python/util/dispatch.py:201: calling softmax (from tensorflow.python.ops.nn_ops) with dim is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "dim is deprecated, use axis instead\n"
     ]
    }
   ],
   "source": [
    "tf.compat.v1.disable_eager_execution()\n",
    "\n",
    "print('Loading model: ')\n",
    "RNN_with_attention_100 = RNN(dropout=0.25)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load a model to train on max 400 node subgraph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading model: \n",
      "WARNING:tensorflow:From <ipython-input-2-7678756ad5a1>:51: GRUCell.__init__ (from tensorflow.python.keras.layers.legacy_rnn.rnn_cell_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This class is equivalent as tf.keras.layers.GRUCell, and will be replaced by that in Tensorflow 2.0.\n",
      "WARNING:tensorflow:From <ipython-input-2-7678756ad5a1>:51: MultiRNNCell.__init__ (from tensorflow.python.keras.layers.legacy_rnn.rnn_cell_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This class is equivalent as tf.keras.layers.StackedRNNCells, and will be replaced by that in Tensorflow 2.0.\n",
      "WARNING:tensorflow:From <ipython-input-2-7678756ad5a1>:56: dynamic_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `keras.layers.RNN(cell)`, which is equivalent to this API\n",
      "WARNING:tensorflow:From /Users/valdimareggertsson/.pyenv/versions/3.8.2/lib/python3.8/site-packages/tensorflow/python/keras/layers/legacy_rnn/rnn_cell_impl.py:561: Layer.add_variable (from tensorflow.python.keras.engine.base_layer_v1) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `layer.add_weight` method instead.\n",
      "WARNING:tensorflow:From /Users/valdimareggertsson/.pyenv/versions/3.8.2/lib/python3.8/site-packages/tensorflow/python/keras/layers/legacy_rnn/rnn_cell_impl.py:570: calling Constant.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
      "WARNING:tensorflow:From /Users/valdimareggertsson/.pyenv/versions/3.8.2/lib/python3.8/site-packages/tensorflow/python/keras/layers/legacy_rnn/rnn_cell_impl.py:580: calling Zeros.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
      "WARNING:tensorflow:From /Users/valdimareggertsson/.pyenv/versions/3.8.2/lib/python3.8/site-packages/tensorflow/python/util/dispatch.py:201: calling softmax (from tensorflow.python.ops.nn_ops) with dim is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "dim is deprecated, use axis instead\n"
     ]
    }
   ],
   "source": [
    "tf.compat.v1.disable_eager_execution()\n",
    "\n",
    "print('Loading model: ')\n",
    "RNN_with_attention_400 = RNN(dropout=0.25)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "# pickle file, saved from running the make_training_set_wikidata notebook\n",
    "current_training_set =  'training_data_rnn_9713_100.pickle'\n",
    "\n",
    "# 'training_data_rnn_39106_10.pickle'\n",
    "\n",
    "with open(current_training_set, 'rb') as handle:\n",
    "    training_data = pickle.load(handle)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Select the first 5000 entities from the full training set, to train on.\n",
    "\n",
    "It had 39106 rows (19503 entities that worked)"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "n_entities = 5000\n",
    "\n",
    "training_data = training_data[:2*n_entities]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "max 400 training set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "current_training_set =  'training_data_rnn_9743_400.pickle'\n",
    "\n",
    "# 'training_data_rnn_39106_10.pickle'\n",
    "\n",
    "with open(current_training_set, 'rb') as handle:\n",
    "    training_data_400 = pickle.load(handle)\n",
    "\n",
    " "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pickle file, saved from running the make_training_set_wikidata notebook\n",
    "   # name has model, number of entities and number of nodes\n",
    "import pickle    \n",
    "current_test_set =  'test_data_rnn_1944_400.pickle'\n",
    "with open(current_test_set, 'rb') as handle:\n",
    "    test_set = pickle.load(handle)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pickle file, saved from running the make_training_set_wikidata notebook\n",
    "   # name has model, number of entities and number of nodes\n",
    "\n",
    "current_test_set =  'test_data_rnn_1939_100.pickle'\n",
    "with open(current_test_set, 'rb') as handle:\n",
    "    test_set = pickle.load(handle)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_chunks(l, n):\n",
    "    return [l[i:i + n] for i in range(0, len(l), n)]\n",
    "\n",
    "def bin_data_into_buckets(data, batch_size):\n",
    "    buckets = []\n",
    "    size_to_data_dict = {}\n",
    "    for item in data:\n",
    "        seq_length = len(item['graph']['vectors'])\n",
    "        question_length = len(item['question_vectors'])\n",
    "        try:\n",
    "            size_to_data_dict[(seq_length, question_length)].append(item)\n",
    "        except:\n",
    "            size_to_data_dict[(seq_length, question_length)] = [item]\n",
    "    for key in size_to_data_dict.keys():\n",
    "        data = size_to_data_dict[key]\n",
    "        chunks = get_chunks(data, batch_size)\n",
    "        for chunk in chunks:\n",
    "            buckets.append(chunk)\n",
    "    return buckets\n",
    "\n",
    "\n",
    "def train_rnn(data, model, saving_dir, name_prefix, epochs=20, bucket_size=10, trace_every=1):\n",
    "    buckets = bin_data_into_buckets(data, bucket_size)\n",
    "    losses = []\n",
    "    \n",
    "    start_time = time.time()\n",
    "\n",
    "\n",
    "    for i in range(epochs):\n",
    "        random_buckets = sorted(buckets, key=lambda x: random.random())\n",
    "        sys.stderr.write('--------- Epoch ' + str(i) + ' ---------\\n')\n",
    "        sys.stderr.write('time_elapsed (s): ' + str(time.time() - start_time) + '\\n')\n",
    "       \n",
    "        for bucket in random_buckets:\n",
    "            graph_bucket = []\n",
    "#             try:\n",
    "            for item in bucket:\n",
    "                node_vectors = item['graph']['vectors']\n",
    "                y = item['answer']\n",
    "                item_vector = item['item_vector']\n",
    "                question_vectors = item['question_vectors']\n",
    "                question_mask = item['question_mask']\n",
    "#                 print('node_vector: {}'.format(type(node_vectors)))\n",
    "#                 print('y: {}'.format(y))\n",
    "#                 print('item_vector: {}'.format(type(item_vector)))\n",
    "#                 print('question_vectors: {}'.format(type(question_vectors)))\n",
    "#                 print('question_mask: {}'.format(type(question_mask)))\n",
    "#                 break\n",
    "                graph_bucket.append((node_vectors, item_vector, question_vectors, question_mask, y))\n",
    "                \n",
    "                \n",
    "            if len(graph_bucket) > 0:\n",
    "                loss = model.train(graph_bucket, 1)\n",
    "                losses.append(loss)\n",
    "#             except Exception as e:\n",
    "#                 print('Exception caught during training: ' + str(e))\n",
    "        if i % trace_every == 0:\n",
    "            save_filename = saving_dir + name_prefix + '-' + str(i) + '.tf'\n",
    "            sys.stderr.write('Saving into ' + save_filename + '\\n')\n",
    "            model.save(save_filename)\n",
    "    return losses, buckets"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "--------- Epoch 0 ---------\n",
      "time_elapsed (s): 0.0006959438323974609\n",
      "Saving into data/rnn-0.tf\n",
      "--------- Epoch 1 ---------\n",
      "time_elapsed (s): 267.9810400009155\n",
      "Saving into data/rnn-1.tf\n",
      "--------- Epoch 2 ---------\n",
      "time_elapsed (s): 511.6649122238159\n",
      "Saving into data/rnn-2.tf\n",
      "--------- Epoch 3 ---------\n",
      "time_elapsed (s): 744.7121930122375\n",
      "Saving into data/rnn-3.tf\n",
      "--------- Epoch 4 ---------\n",
      "time_elapsed (s): 977.31556391716\n",
      "Saving into data/rnn-4.tf\n",
      "--------- Epoch 5 ---------\n",
      "time_elapsed (s): 1206.830137014389\n",
      "Saving into data/rnn-5.tf\n",
      "--------- Epoch 6 ---------\n",
      "time_elapsed (s): 1435.1079289913177\n",
      "Saving into data/rnn-6.tf\n",
      "--------- Epoch 7 ---------\n",
      "time_elapsed (s): 1666.308042049408\n",
      "Saving into data/rnn-7.tf\n",
      "--------- Epoch 8 ---------\n",
      "time_elapsed (s): 1900.0522849559784\n",
      "Saving into data/rnn-8.tf\n",
      "--------- Epoch 9 ---------\n",
      "time_elapsed (s): 2143.0867908000946\n",
      "Saving into data/rnn-9.tf\n",
      "--------- Epoch 10 ---------\n",
      "time_elapsed (s): 2395.67049908638\n",
      "Saving into data/rnn-10.tf\n",
      "--------- Epoch 11 ---------\n",
      "time_elapsed (s): 2649.23659491539\n",
      "Saving into data/rnn-11.tf\n",
      "--------- Epoch 12 ---------\n",
      "time_elapsed (s): 2898.0353178977966\n",
      "Saving into data/rnn-12.tf\n",
      "--------- Epoch 13 ---------\n",
      "time_elapsed (s): 3148.362883090973\n",
      "Saving into data/rnn-13.tf\n",
      "--------- Epoch 14 ---------\n",
      "time_elapsed (s): 3399.276978969574\n",
      "Saving into data/rnn-14.tf\n",
      "--------- Epoch 15 ---------\n",
      "time_elapsed (s): 3652.906282901764\n",
      "Saving into data/rnn-15.tf\n",
      "--------- Epoch 16 ---------\n",
      "time_elapsed (s): 3952.8570749759674\n",
      "Saving into data/rnn-16.tf\n",
      "--------- Epoch 17 ---------\n",
      "time_elapsed (s): 4219.476907014847\n",
      "Saving into data/rnn-17.tf\n",
      "--------- Epoch 18 ---------\n",
      "time_elapsed (s): 4457.760031938553\n",
      "Saving into data/rnn-18.tf\n",
      "--------- Epoch 19 ---------\n",
      "time_elapsed (s): 4700.630737066269\n",
      "Saving into data/rnn-19.tf\n",
      "--------- Epoch 20 ---------\n",
      "time_elapsed (s): 4926.099878072739\n",
      "Saving into data/rnn-20.tf\n",
      "--------- Epoch 21 ---------\n",
      "time_elapsed (s): 5151.372601032257\n",
      "Saving into data/rnn-21.tf\n",
      "--------- Epoch 22 ---------\n",
      "time_elapsed (s): 5374.425382137299\n",
      "Saving into data/rnn-22.tf\n",
      "--------- Epoch 23 ---------\n",
      "time_elapsed (s): 5597.247031927109\n",
      "Saving into data/rnn-23.tf\n",
      "--------- Epoch 24 ---------\n",
      "time_elapsed (s): 5825.6189839839935\n",
      "Saving into data/rnn-24.tf\n",
      "--------- Epoch 25 ---------\n",
      "time_elapsed (s): 6054.85258102417\n",
      "Saving into data/rnn-25.tf\n",
      "--------- Epoch 26 ---------\n",
      "time_elapsed (s): 6280.5028212070465\n",
      "Saving into data/rnn-26.tf\n",
      "--------- Epoch 27 ---------\n",
      "time_elapsed (s): 6501.212959051132\n",
      "Saving into data/rnn-27.tf\n",
      "--------- Epoch 28 ---------\n",
      "time_elapsed (s): 6720.426610946655\n",
      "Saving into data/rnn-28.tf\n",
      "--------- Epoch 29 ---------\n",
      "time_elapsed (s): 6942.410737037659\n",
      "Saving into data/rnn-29.tf\n",
      "--------- Epoch 30 ---------\n",
      "time_elapsed (s): 7161.8735411167145\n",
      "Saving into data/rnn-30.tf\n",
      "--------- Epoch 31 ---------\n",
      "time_elapsed (s): 7380.357953071594\n",
      "Saving into data/rnn-31.tf\n",
      "--------- Epoch 32 ---------\n",
      "time_elapsed (s): 7600.412356138229\n",
      "Saving into data/rnn-32.tf\n",
      "--------- Epoch 33 ---------\n",
      "time_elapsed (s): 7819.461009025574\n",
      "Saving into data/rnn-33.tf\n",
      "--------- Epoch 34 ---------\n",
      "time_elapsed (s): 8038.039269208908\n",
      "Saving into data/rnn-34.tf\n",
      "--------- Epoch 35 ---------\n",
      "time_elapsed (s): 8282.68532705307\n",
      "Saving into data/rnn-35.tf\n",
      "--------- Epoch 36 ---------\n",
      "time_elapsed (s): 8523.235234975815\n",
      "Saving into data/rnn-36.tf\n",
      "--------- Epoch 37 ---------\n",
      "time_elapsed (s): 8737.588361024857\n",
      "Saving into data/rnn-37.tf\n",
      "--------- Epoch 38 ---------\n",
      "time_elapsed (s): 8949.135949134827\n",
      "Saving into data/rnn-38.tf\n",
      "--------- Epoch 39 ---------\n",
      "time_elapsed (s): 9162.365429878235\n",
      "Saving into data/rnn-39.tf\n",
      "--------- Epoch 40 ---------\n",
      "time_elapsed (s): 9377.573159217834\n",
      "Saving into data/rnn-40.tf\n",
      "--------- Epoch 41 ---------\n",
      "time_elapsed (s): 9594.063915014267\n",
      "Saving into data/rnn-41.tf\n",
      "--------- Epoch 42 ---------\n",
      "time_elapsed (s): 9805.972912073135\n",
      "Saving into data/rnn-42.tf\n",
      "--------- Epoch 43 ---------\n",
      "time_elapsed (s): 10013.28000998497\n",
      "Saving into data/rnn-43.tf\n",
      "--------- Epoch 44 ---------\n",
      "time_elapsed (s): 10234.11480998993\n",
      "Saving into data/rnn-44.tf\n",
      "--------- Epoch 45 ---------\n",
      "time_elapsed (s): 10452.216536045074\n",
      "Saving into data/rnn-45.tf\n",
      "--------- Epoch 46 ---------\n",
      "time_elapsed (s): 10660.503604888916\n",
      "Saving into data/rnn-46.tf\n",
      "--------- Epoch 47 ---------\n",
      "time_elapsed (s): 10869.056018829346\n",
      "Saving into data/rnn-47.tf\n",
      "--------- Epoch 48 ---------\n",
      "time_elapsed (s): 11077.74135184288\n",
      "Saving into data/rnn-48.tf\n",
      "--------- Epoch 49 ---------\n",
      "time_elapsed (s): 11285.81407403946\n",
      "Saving into data/rnn-49.tf\n",
      "--------- Epoch 50 ---------\n",
      "time_elapsed (s): 11492.625254154205\n",
      "Saving into data/rnn-50.tf\n",
      "--------- Epoch 51 ---------\n",
      "time_elapsed (s): 11701.713317155838\n",
      "Saving into data/rnn-51.tf\n",
      "--------- Epoch 52 ---------\n",
      "time_elapsed (s): 11908.945063829422\n",
      "Saving into data/rnn-52.tf\n",
      "--------- Epoch 53 ---------\n",
      "time_elapsed (s): 12115.278746128082\n",
      "Saving into data/rnn-53.tf\n",
      "--------- Epoch 54 ---------\n",
      "time_elapsed (s): 12320.817554950714\n",
      "Saving into data/rnn-54.tf\n",
      "--------- Epoch 55 ---------\n",
      "time_elapsed (s): 12527.153759002686\n",
      "Saving into data/rnn-55.tf\n",
      "--------- Epoch 56 ---------\n",
      "time_elapsed (s): 12740.531570911407\n",
      "Saving into data/rnn-56.tf\n",
      "--------- Epoch 57 ---------\n",
      "time_elapsed (s): 12955.444499015808\n",
      "Saving into data/rnn-57.tf\n",
      "--------- Epoch 58 ---------\n",
      "time_elapsed (s): 13170.674001932144\n",
      "Saving into data/rnn-58.tf\n",
      "--------- Epoch 59 ---------\n",
      "time_elapsed (s): 13379.902307033539\n",
      "Saving into data/rnn-59.tf\n",
      "--------- Epoch 60 ---------\n",
      "time_elapsed (s): 13590.915923833847\n",
      "Saving into data/rnn-60.tf\n",
      "--------- Epoch 61 ---------\n",
      "time_elapsed (s): 13798.552588939667\n",
      "Saving into data/rnn-61.tf\n",
      "--------- Epoch 62 ---------\n",
      "time_elapsed (s): 14008.035825014114\n",
      "Saving into data/rnn-62.tf\n",
      "--------- Epoch 63 ---------\n",
      "time_elapsed (s): 14216.87078499794\n",
      "Saving into data/rnn-63.tf\n",
      "--------- Epoch 64 ---------\n",
      "time_elapsed (s): 14424.43907403946\n",
      "Saving into data/rnn-64.tf\n",
      "--------- Epoch 65 ---------\n",
      "time_elapsed (s): 14631.78543305397\n",
      "Saving into data/rnn-65.tf\n",
      "--------- Epoch 66 ---------\n",
      "time_elapsed (s): 14839.367418050766\n",
      "Saving into data/rnn-66.tf\n",
      "--------- Epoch 67 ---------\n",
      "time_elapsed (s): 15047.84737610817\n",
      "Saving into data/rnn-67.tf\n",
      "--------- Epoch 68 ---------\n",
      "time_elapsed (s): 15257.174207925797\n",
      "Saving into data/rnn-68.tf\n",
      "--------- Epoch 69 ---------\n",
      "time_elapsed (s): 15467.439636945724\n",
      "Saving into data/rnn-69.tf\n",
      "--------- Epoch 70 ---------\n",
      "time_elapsed (s): 15674.977701187134\n",
      "Saving into data/rnn-70.tf\n",
      "--------- Epoch 71 ---------\n",
      "time_elapsed (s): 15883.257030010223\n",
      "Saving into data/rnn-71.tf\n",
      "--------- Epoch 72 ---------\n",
      "time_elapsed (s): 16093.164541959763\n",
      "Saving into data/rnn-72.tf\n",
      "--------- Epoch 73 ---------\n",
      "time_elapsed (s): 16302.148610115051\n",
      "Saving into data/rnn-73.tf\n",
      "--------- Epoch 74 ---------\n",
      "time_elapsed (s): 16515.929139852524\n",
      "Saving into data/rnn-74.tf\n",
      "--------- Epoch 75 ---------\n",
      "time_elapsed (s): 16731.209547042847\n",
      "Saving into data/rnn-75.tf\n",
      "--------- Epoch 76 ---------\n",
      "time_elapsed (s): 16943.18325805664\n",
      "Saving into data/rnn-76.tf\n",
      "--------- Epoch 77 ---------\n",
      "time_elapsed (s): 17153.767283916473\n",
      "Saving into data/rnn-77.tf\n",
      "--------- Epoch 78 ---------\n",
      "time_elapsed (s): 17363.097918987274\n",
      "Saving into data/rnn-78.tf\n",
      "--------- Epoch 79 ---------\n",
      "time_elapsed (s): 17573.928687095642\n",
      "Saving into data/rnn-79.tf\n",
      "--------- Epoch 80 ---------\n",
      "time_elapsed (s): 17784.849951982498\n",
      "Saving into data/rnn-80.tf\n",
      "--------- Epoch 81 ---------\n",
      "time_elapsed (s): 17995.553105831146\n",
      "Saving into data/rnn-81.tf\n",
      "--------- Epoch 82 ---------\n",
      "time_elapsed (s): 18207.953310012817\n",
      "Saving into data/rnn-82.tf\n",
      "--------- Epoch 83 ---------\n",
      "time_elapsed (s): 18421.098495960236\n",
      "Saving into data/rnn-83.tf\n",
      "--------- Epoch 84 ---------\n",
      "time_elapsed (s): 18631.334619045258\n",
      "Saving into data/rnn-84.tf\n",
      "--------- Epoch 85 ---------\n",
      "time_elapsed (s): 18840.672689199448\n",
      "Saving into data/rnn-85.tf\n",
      "--------- Epoch 86 ---------\n",
      "time_elapsed (s): 19050.42599081993\n",
      "Saving into data/rnn-86.tf\n",
      "--------- Epoch 87 ---------\n",
      "time_elapsed (s): 19258.643491983414\n",
      "Saving into data/rnn-87.tf\n",
      "--------- Epoch 88 ---------\n",
      "time_elapsed (s): 19466.458351135254\n",
      "Saving into data/rnn-88.tf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "--------- Epoch 89 ---------\n",
      "time_elapsed (s): 19674.66388988495\n",
      "Saving into data/rnn-89.tf\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 90\n",
    "\n",
    "_saving_dir = 'data/'\n",
    "losses_ru, bucket = train_rnn(training_data,\n",
    "      RNN_with_attention_100,\n",
    "      _saving_dir,\n",
    "      name_prefix='rnn',\n",
    "      epochs=n_epochs,\n",
    "      bucket_size=10,\n",
    "      trace_every=1,\n",
    "      )\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "trained on max 400 nodes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "--------- Epoch 0 ---------\n",
      "time_elapsed (s): 0.002198934555053711\n",
      "Saving into data/rnn-0.tf\n",
      "--------- Epoch 1 ---------\n",
      "time_elapsed (s): 1212.5825307369232\n",
      "Saving into data/rnn-1.tf\n",
      "--------- Epoch 2 ---------\n",
      "time_elapsed (s): 2435.6520409584045\n",
      "Saving into data/rnn-2.tf\n",
      "--------- Epoch 3 ---------\n",
      "time_elapsed (s): 3659.8572828769684\n",
      "Saving into data/rnn-3.tf\n",
      "--------- Epoch 4 ---------\n",
      "time_elapsed (s): 4933.030325889587\n",
      "Saving into data/rnn-4.tf\n",
      "--------- Epoch 5 ---------\n",
      "time_elapsed (s): 6168.8119328022\n",
      "Saving into data/rnn-5.tf\n",
      "--------- Epoch 6 ---------\n",
      "time_elapsed (s): 7406.63069486618\n",
      "Saving into data/rnn-6.tf\n",
      "--------- Epoch 7 ---------\n",
      "time_elapsed (s): 8582.185778856277\n",
      "Saving into data/rnn-7.tf\n",
      "--------- Epoch 8 ---------\n",
      "time_elapsed (s): 9739.916426897049\n",
      "Saving into data/rnn-8.tf\n",
      "--------- Epoch 9 ---------\n",
      "time_elapsed (s): 10905.071465015411\n",
      "Saving into data/rnn-9.tf\n",
      "--------- Epoch 10 ---------\n",
      "time_elapsed (s): 12128.376188993454\n",
      "Saving into data/rnn-10.tf\n",
      "--------- Epoch 11 ---------\n",
      "time_elapsed (s): 13475.74343085289\n",
      "Saving into data/rnn-11.tf\n",
      "--------- Epoch 12 ---------\n",
      "time_elapsed (s): 14877.980217695236\n",
      "Saving into data/rnn-12.tf\n",
      "--------- Epoch 13 ---------\n",
      "time_elapsed (s): 16282.42160487175\n",
      "Saving into data/rnn-13.tf\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 14\n",
    "\n",
    "_saving_dir = 'data/'\n",
    "losses_ru, bucket = train_rnn(training_data_400,\n",
    "      RNN_with_attention_400,\n",
    "      _saving_dir,\n",
    "      name_prefix='rnn',\n",
    "      epochs=n_epochs,\n",
    "      bucket_size=10,\n",
    "      trace_every=1,\n",
    "      )\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "c) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "max 10 nodes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " 10 epochs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x3928e3be0>]"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAD4CAYAAAD1jb0+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deZgU1bk/8O/LDgICMiIiZNC4hMS4ZKImrjFqXJKY3MSrJjdXEw3eJOZnEnN/ITFGjCYILjEao2JE0CiuuLEKw7AICA47s8DAMAMMs7PMPtPTfe4fXTP09HR1V1fX2v39PA8PPdXVVW91Vb916tSpc0QpBSIi8p8+bgdARETmMIETEfkUEzgRkU8xgRMR+RQTOBGRT/VzcmWjR49W2dnZTq6SiMj3Nm7cWKeUyoqe7mgCz87ORn5+vpOrJCLyPREpjzWdVShERD6VMIGLyHgRyRORQhEpEJF7tOlTRaRCRLZo/663P1wiIupipAqlE8C9SqlNIjIMwEYRWaq99zel1GP2hUdERHoSJnClVCWASu11o4gUARhnd2BERBRfUnXgIpIN4DwA67VJd4vINhGZJSIjdT4zWUTyRSS/trY2pWCJiOgYwwlcRIYCeAfAr5RSDQCeBXAagHMRLqE/HutzSqmZSqkcpVROVlavVjBERGSSoQQuIv0RTt6vKqXmAYBSqlopFVRKhQC8AOAC+8IkIqJoRlqhCIAXARQppZ6ImD42YrbvAthhfXhE/tDU3on3Nle4HQZlGCOtUC4G8CMA20VkizbtDwBuFZFzASgAZQDusiVCIh/447vb8d6Wg8gefRzOHT/C7XAoQxhphfIxAInx1kLrwyHyp6qGNgBAS0eny5FQJuGTmEREPsUETkTkU0zgREQ+xQROuhraAlhSUOV2GESkgwmcdP3mja2465WNKKtrdjsUIoqBCZx07T/UAgBo6wy6HAkRxcIEThnvN29uwT2vb3Y7DKKkMYFTxpu3qQLvbznodhhESWMCJyLyKSZwSkgptyMgoliYwEmXxOpAIYaOzhBCIWZ5IqcxgVPKzvjjItzzxpbEMxKRpZjAyRIfbuVNQCKnMYETEfkUEzgl5LWbmDkPL8NPX853O4zYPPZdUXpjAiffqWtqx9LCarfD6EFidplPZC8mcJ/bfuAoCg82uB0GEbnAyJBq5GHf+sfHAICyR25wORIichpL4EREPsUETgkp3pkj8iQmcIe0dwZxxn2L8P6WCrdDMUyMPopJPMmRK5jAHVLf1IGOYAiPLCp2OxSyE8955CAmcCIin2ICJyLyKSZwSshrT2ISURgTeBLaO4NoC2TO+JCsziXyNibwJHzpoWU46/7FbodBRASACTwpTe2dbodARNSNCZyIyKeYwAFM/aAANzy12u0wKB2YvOFbVteMIIeloyQxgQOYvbYMBezRj1KQSneye2qbcMVjK/BUbomFEVEmYAInXXyS3hlVR9sAAJ+WHXI5EvKbhAlcRMaLSJ6IFIpIgYjco00fJSJLRaRE+3+k/eGSHy0vrsaMxexCgMhqRkrgnQDuVUpNAnARgF+IyCQAUwDkKqVOB5Cr/U0J+OmhGKti/cnsfPxzxR5rFkZE3RImcKVUpVJqk/a6EUARgHEAbgQwR5ttDoDv2BVkOmB1hDdlYrXFgcMt2FXd6Ph6Cw824OypS1DT2Ob4utNVUnXgIpIN4DwA6wGMUUpVam9VARij85nJIpIvIvm1tbUphEpkvZueW+d2CI67ZHoervnbKsfX+9KavWhs68SKYuYBqxhO4CIyFMA7AH6llOrRZEMppaDTgEopNVMplaOUysnKykopWHIWrxqIvM1QAheR/ggn71eVUvO0ydUiMlZ7fyyAGntCJCKiWIy0QhEALwIoUko9EfHWBwBu017fBuB968MjIj+45/XNmPpBgdthZBwjJfCLAfwIwJUiskX7dz2ARwBcLSIlAK7S/iYizcvrypA9ZQFaO9K/B8v3txzE7LVlboeRcfolmkEp9TH0exb9urXhkBf5qemjl/wzL9x08khrBwYPGOxyNJnr/S0VmLthH16f/BW3Q7FcwgROmYs3MckOTg8Afc/rWxxdn5PS9lH6Q80dONoScDsMyhAclT4xFgisl7Yl8PMfWgoAKHvkhh7TQ1qPb336OHs0sRoiQ6RwWPEYoWSlbQlczzkPfoRLZ+S5tn6WQigaDwkyK21L4Hoa2zvRyJF1ksLqgSTwqyIHZVwJ3G1+ukxOpY/rTBPru+KJj+zGBB7Bzg5+WHWSuXgi7MlPhRivYwKPUN3gXi9pNQ1tmL64uPsmK1G64YnMekzgHvHbt7fh2RV7sCEDuzf1q/bOIPKK2QUQuYcJ3CM6OsOPW4c8eH3pwZA84a8LivDj2Z9i877DbodCGYoJPMPc9Nxaw4Pnul1vv7SwGtsPHE36c4t3VCF7ygLUN7XbENUxZfUtAIAjrXxgzC8Wba9EblG122FYhgk8zfzmzS2Y/HK+7vuflh3GE0t3xV1GIBjCntomq0NL2k9fzse3/vFx0p+bvXYvAGCnC6PORDJ65cILHOf87NVNuGOO/u/DbzKuHbjnpfhrnrepIuUQ/rKgCLPXliFr2MCUl5Vuahvbsau6ERd/drThzxi9knH7iscpPGFZhyVwF72z8QCufHyF22H08klpPQDgSEuHy5GEHWnpwJWPrXBlHMdo33t2LX74r/Vuh+FLmXKCWl5cjewpC3DwSKvt62ICt0jFkVbMXrM34XxVEU0V731rK0prm3vO4MGD3O0S04qdtSita8YzebtdjgTYd6jFtmWn+83i5gzoFx0AXt+wHwCwTbt/s2pXLS5/NA/tndZvPxO4RW6btQFTPyxEbWPiG2dmbsy5ge12nZEp3/KHWw/qvtfaEcS9b27FoWZvXPVZaeoHBSivb8GBw9aXyH2fwAPBUPclf01DG3KLqnHJ9OWmlpVKCeio1hJBGVhIY3ucVgtpXgpLS9xnKXtr4368s+kAnli60+1QfMX3CfyxJTtxy8xPsHX/Edw88xPcMSffljOdGbfMXIcvPLDE0Lx2lnZ3VjVi6gcFhk4udnJ7/VZLVKdr99YGgiF0BkMxp09bWOTL/vB51Zcc3yfwrhtb9c3t2FvXnGBuZ31SeghNBns+tLPjo9tf2oDZa8tQedS9rgIyUdf5KtmUtLumydAN5M/dvxiXTO/dNfLC7ZV4flUp/rKwMMk1OyPNzuOu8n0C9wvDJQsPFkDcLjn7vle/JPfpVU+sxHV/X51wvs6Q6nFTvEtQ608nELTveyura8baPXW2Ld/PnDxa2Q48jv2HWtC/bx+cdPwg51bq81xlJw+e23qxavd5/WrpisdWAOg94pURvj8hG+REs0nfl8BrtFYfO6usf3Lw0hl5uGhabq/pR1sD+LjE2tKHl+v+xKIj0fGCvIPry5Q2zl6z/1ALPs3gDuB8n8ALDjYAAOasLXNsnXe9ko//enG9L28S+ZnRfGzHybC+qd1w9wLZUxZg2sIiy2PIBMmWzi+dkYebnlsHACg82IDDHmyGaGc5glUoJuyuCf+QO2K0ADDLi5eVXonJC1cnlz+6Iu4N6eiri+dXlWL00PTqiqAtEMT0xcX49dVndE/7YOtBHDegb8z51+yuw8B+fZCTPSrhsq3Yw9c/tRrZJwzBiv/9mgVL8wcmcIcYTobu5yrLqkyimT0deOFEEjd5R8Tngd1nm7fy9+OlNWU9Tqj/b+5m3fm7uhzQqyf/84eFaGwL4NGbzrEsxq4eIr3EzmPC91UodnGt5YX7uaoXo9/F5Y/m4UYTvQca5bXkmDiexN9bIBjC5v1HrAin99otPoa7WrcEQ6ldeXaFNWvNXry18UCqYfUSCik8lVviyeoUq7EEruPKx1c6uj4vVBNEC8dkPAmU17eg3L5wfCvevp2+qBj/+jhxHzpJrc97h5Kj1uypwxNLd2FnVSOe+eH5bodjK5bAdXjtoaBIjW0BBCysf09WU3snns4t6S6RGeV2e3IvKqxs6H7thaoiIFzSnr64WLdfkg4b25dboeu30dJh7CE6O7UFgii1MZf4OoHblRA+KqiyZbnxJPrxflRQhU3a0F1nT/0IP/v3RhPrsMa0hUV4fOkuLNpRadES/cnbacy8lbtq8OyKPbj/vR0x358fp1OqVKXbOf7AYXvr5H2dwI8k2YzPaE9nk1/pmRyrjrbhyWW9R7F5b3PqgydEa+4I4q8Li9AW6Nn15ORXNuI//rm2++9lRcYH09W7oq482opX1ydf6dGidQva0WntVcA/lpcge8qCXlcXXv9Re7H6KxWdWgm7XWf/Bu3YIWlU7+Pk8Zo2deBGLj9fMtBfdyy/nLsJn5b1Hrj2LwuLcNkZWTjzpGGmlhupKwn8I283tu4/gpOGD8JPLpmY8nLjuX3Wp9hZ3YhrJp0Ud/Qdp47H51aWAghfdvbv27tsYVfrGCI7RB+tdhy9CUvgIjJLRGpEZEfEtKkiUiEiW7R/19sQm61CIdWrg/Xy+th1Va0B/Y7Yo0vKRsQrsQW0Uo9e/fK0RdY9IHJI6zBJtyrK4iMu1ROB3+rQfRauY8x+LW2BIG6d+QmKqxoSz+xBdhwORqpQZgO4Nsb0vymlztX+LbQ2LPv99u2tOPOPi3tMu//9AtvWZ9Vl9vNaKTUWp2/aLNrhzL0CswVvp/Jnoqs/vfjbAkGU29huOd3OHxvLD2NdaT3+/KG3elmsbmjDPpfanydM4EqpVQA839lAogSplOqR/FIb/NfaoqlVJcs/6tx0ssvSwmrsqPDH6EJ2SOYoqDza2ms/3/3aZlREjJsY7xhuCwQxbWGRoZO0dXXysY9LXln0dOFfc3HZo3nYb+Nwe3pSuYl5t4hs06pYRurNJCKTRSRfRPJra2tTWF1vkcdRc4J+t/fUNln66HuX/PLDOP2+hd1Dqbl5bCd7AFnxQ2xJYpxDs+vr+tx7Ww7iqieOtc9fXRL/ePJKjfmemiZ8ZdpyzFzV8+opUfyR/v1JOZ5fVYp/5u2xOrxe9O41OHEPws/nhn+baBCQKrMJ/FkApwE4F0AlgMf1ZlRKzVRK5SilcrKyskyuLrHGBAm8M8k2y0Y9NL/Qln6X7W4TXNPYhrqm8EnnfxI0SUwl0edb3FNcVz80ALDVpicYrbZfGyFq7Z5608voOsYCKT4F6VVdp4bX1u9LOC+vAI4xlcCVUtVKqaBSKgTgBQAXWBuWtyRTykyVUw0tvv30mu7Xm/bFToRWhPJ9rae4TOGF3OKVB4Ks5pUrqi67a5qQPWWBqzGYSuAiMjbiz+8CcLby1WGltT1bp3SVXGMxcpDF+4E5VbqINZKLV/gx/cQ68bq9HXYlvJRPEB4sQucWVeN7z65FKIkr9aWF1Umtw479kbAduIjMBXAFgNEicgDAAwCuEJFzET5GywDcZUNscXUGQ6j2cBLyiqOtge66f7eb4TW0BeJ2seq1Elaqur7vdNsuO7id0u9+bTNaA0G0dQYxZIB/Ho9JGKlS6tYYk1+0IZaE8oprsHJXLaZ++/P468JizDL5YI4XrSsN14+mUoUSq/XBOQ9+ZH6BFnvggwI884PkOxfSO+8YPR9tLD+EzqDChaeekPS6jdCLo6EtfF9Gb5+aSlpJfCj1dvdAxZFWHG7uwBfGHZ/i0lLTFgjiba3nQj9VEUUeG261A/eMH8/+FLO1kXeWFyd3+eKU9s5Q0p08OcV8KwJrtidRSyG7fO/Zdbh55ieWLzfR9xl9HCT6FuMlpmR2XarNCCM/ffEjy/HNp+3rItios+5fjHkWdl0xbVFRrwf5rOPc799XCTwV1Q369dZWunRGHu56ZaOpJzQjeaGa0Oobqma3SS8OvzxZP39benT6ZWcVXLxdaceNwudXluKNT/dbvtxITjS79EVlT31TO+pT7Jz9zjmfWhRNYsuKqrEgTX60fuaBcyAAYHmx8Y7H/MDtzrsSnUdeXmesPXZnRPNfP1XLRPJFCfzhBUW45m+rUlqG0bbaj3+0M6X1dLn3ra2G53Ws5cnRVt33fvHqpl6X/G0B+9ocH2ruwI9f2hBz1JTbZm3AM3m7Ey5DtwsXn5TMzeoMhlAfpyWU3VJNdnYf7it2mn9gMPLktKu6EdlTFiC3KHZ1ba/jzIVzgC8S+Ls2dNuq5+nliROHncwmnyeX7UJbgjq97z2r3yZ7wfZK7HPwUeDZa/Yib2dtzNLS5n1H8OiSYyfSVE5wNY3p1VJJAfjz/EJ86eFljq/bqiqBDXud6ZnD6GHzyiflMQsrm7X+95e4MD6AUb6oQknFu5sPoL7J/bHxoo99q/vSfnJZiaXLixQvgS4tNH5wu3GR+oCNHZSlKpljIPLwWWxRJ2JKKfzh3R24KecUnD9BtzcMy6V6f8hqegNXWM2V7mT97tdvbMXDC6zrgtWsyCQoEPzh3e3x509xfUdbA6Zv/iRzoOWX9+4nPRmHmiOqApI8wqNPio1tATwVNdRbMqX3ffUtuPbJVaaqJ5y86dzHYEk4UUytgSDmbtiHH76w3oKojEv1uzL6cS/UpNn9FHfaJ3CvWrO7LuE8SiksK6xO6umwLm70jJaMri2aE1GFkuoP7pFFxXhi6S6sN3mJ/sLqUhRXNWLB9uRuQPc8OcebLzxjeX0zPkxhWLI+USt5bmXPDq7svgfghRZSdunqI98qNz5zrMuKjG8H7lcPfliAi6blJv25dzZV4M6X8/HahsQd/PiB1c3QohfnZJ81qbju76vxy7mbTX8+ui76kUXFqYbkushN2lh+CAePtOL9LRWOPz0c60a/l09YaV8H7gUvrSkz9bmug6lSp/XI62mS2P3OaIl3dUn4qiuVE41eQvvZvzdixve/iGGD+hteVte9IaOtSrrqru0u4UfebL/n9S22rSfRVidqLtnrXRfqbDK+BN7QltzAyFbR29ddv08jx8KUefr16F67UeSWdGlSmGg7Fu2oSvrBlEtn5HW/Xl1Siy88sARNEU/LhqJOFtMsKumnWqB1qhWLH2R8CfyLU73TV4iVnOrGNdVBGtLND/7l7A1Bqzz20S40tXdixuLi7qadeSm0p3baWgP3lJzCUenT3OIdPW+S3T6791Oi0xYVd5e6nDog2juDPQZMMGLbAXODKljx5FvBwYbuljbfOudk9LWotO3WycXNk1rXV2f0KUarTU+xdO+FE+fzK0txxyUTu//2xKj0XnS4uQNlLg0iaoU568px8OixB0xW7TpW0tlbd6zvcad/0P/71jZc9cTKHpfRicRqIDNtoTPNNldGfG8fbj2I2qjmf7G+v1kf78UvXt0Uc3kp9QRp8LMvrNIflDoeo8eCUqq7F8Rklp3MtttxXCYbs11ibVsym7ttv7NjxPoygUfXzaUTN1tSbNcGKE61/vx5k0lK70k/o3s7VncJ0Yv88/zCpJsJJpLM4fgXm09us9aUdT+Ysn5vPaYvNlaytbJ0WHGkFbfM1K/CS6aAEM8r68osWY4uH9w/8WUCT9/07U1Wfd89+ka2YKHRydmq31tdUzvWGRi/0urft14peHVJLYqrGg0t41+rj508qxva8ewKY4MgW9lz3tO5JfikVP9Go1U3Ie9P8inb+dvMt703Kt5hzXbgmq6O3ckZLR1BtHQkV2q6/aUN3X1JdPFLj29PL9+NW1+wvv9wM/bUNuFHL24w3B9Q5VFzfb/4oLCZsk3lie/XWHEec/Io9+VNzHR4cCEZbqe922ZtAACUPXKD4c+s2FmLfQbvU5TVNSN79HGmYosU/T2lQxPCrz++0vC8plsEIck6cHOrIRv4sgSeaRpa3WmrbqfIJHDFYyvizutUHvZ7vte7wom8Sa7Hqj6+//T+DlQc0e+22E1GrgBj3sRM4oy1aHulo8eRL0vgmebV9Xzi0iy3Bx+wmplS9sxVpbjsjKz4MyX4mib9abGhWd1qhui0UEhhW0XvFifzNldg3IjBMT/DZoSUNqITUWuc1jd6OcvIDyJeqeulqEGxvVY1cCjGYBdLdQYXAIDcIvMj/yT6Lv3Sz0w8Zk/msaqXXlhdqjvqlpNXICyBkyd8LqKEZ1RzgqTy2vp9WLhdv+/sBz8sTHqdTjr/oaW9ph1u7tDt78RsL4xIsh24XxmpQjH6PRhtFWQ3JnAyrDMYwvOrSuOWlnuI+jH0aEaYRHm3pMbcj8Xq9t56vFZyN8PpqqY75+Tbvo7o/WK2Uzmz5jrQ2RwTOPVS0xh7QIN3Nh3oMdSZGW2BIM66P7nS9vMrzT0YZDe/lFpbA0GU1jbh1KyhuvM43QplWZyqICPM9JEf7aH5hT0efQf0b2IGQwqdoRAG9utrePm5UYNZsx04ucpwyTsOL9el+iEfB4Iq6TrWjeWHceXjK9EZ1B/CzS8noy4rDbSsSVXXV9IRDOHHsz/FmX9MvprPbkzgZNihltSaMyoF9LeqxykbNLW7e3Ix0sKkNYVuDuIt3s3WOma6bgjEORlZ7cOtB7ubYnqtD34mcDLsqdzUB0628pHtaKk+ov3Opp5P+GZPWYBn8nantEy/cLMEbtd3bMcmTZm3HW2BoOEnY+3GBE62if4BrStN3L+I17yi0645FFJYlkKzPV9LsjI3e8oCfBJn35vp3MrNG8dmmwmyHTj5npuXoJGdGZVUp9YMLBA6dgnv3UqhnvSqaJzoo2bqB8l1POUWv7UoYgIn28T6MTy8wJm+wmPZduDYk3NX/21VSsvq2STS/5ot6uI1nehV9933rv5Qhk5jAifbHI7xJKHf+KUHxVRt2pfEyEomLjms6gM8GUds6kMoXle5TkuYwEVklojUiMiOiGmjRGSpiJRo/4+0N0zyo8MptlrJRHqjzvvdgcP69cZ2tYDZvO8IjqZhR3CRjJTAZwO4NmraFAC5SqnTAeRqfxN52kILn8yMzLM/fTkfaw0MAGGEnSOu611NpOk5A4C3evJ05UEepdQqANFH1Y0A5miv5wD4jsVxEVkuXikwWdUN5gZOSKQ5yYEzXOGBhG/VSadrUGyrl+sUs4/Sj1FKdRVnqgCM0ZtRRCYDmAwAEyZMMLk6IndUN7Rjy/7e9cO/e2ebLeubv9W+/luqj7Zj8/7DiWckBIIhFFc2WLpMOyqKUu4LRSmlRET3vKWUmglgJgDk5OT47PxGBHznmTW9pnVa0BdHLPNsfEDkskfzYk63a1u8wOwDSo8sKsaLH+9NPKPLzLZCqRaRsQCg/Z+hTzRQpkrXm41uMJdkjX3/ZndTrKuuVK0uqbN8mWYT+AcAbtNe3wbgfWvCIcpcfutQyg8unRH7qkNPZ8i+PlbqbWhWa6QZ4VwA6wCcKSIHROQOAI8AuFpESgBcpf1NlDHsKH+zUG/c//x7ky3L/bM2yEfQJ9VKCevAlVK36rz1dYtjIfINJtv0VFzViF3VjbZUodiBT2ISmZDJ+bvD4q5cvVZzdE2K3SzosuGszwRORK7KlLp/jshDRGnHwbEZXMXuZIk8YqtP6kj9oLHNO4+7+w0TOBGRA1iFQkRpJ5NvCKeKCZzII9bb2BOhl7FJpnlM4ETkqkzploA3MYko7WRG+mYdOBGloUwpgduBCZyIXMX0bR4TOBG5qq6p3e0QfIsJnIhctbE8M0YJsqOmiAmciFxl16j0mYAJnIhclSmdWdmBCZyIXNWHGdw0JnAichXzt3lM4ETkqsa2TrdD8C0mcCIin2ICJyJygLLhkSUmcCIin2ICJyLyKSZwIiKfYgInIvIpJnAiIgewLxQiIurGBE5E5AA7njhlAicicgCrUIiIqBsTOBGRTzGBExH5VL9UPiwiZQAaAQQBdCqlcqwIioiIEkspgWu+ppSqs2A5RESUBFahEBE5wIZGKCkncAXgIxHZKCKTrQiIiIiMSbUK5RKlVIWInAhgqYgUK6VWRc6gJfbJADBhwoQUV0dERF1SKoErpSq0/2sAvAvgghjzzFRK5SilcrKyslJZHRERRTCdwEXkOBEZ1vUawDUAdlgVGBERxZdKFcoYAO9K+AH/fgBeU0ottiQqIqI0Y0NXKOYTuFKqFMA5FsZCRJS2vNgKhYiIXMIETkTkAPZGSERE3ZjAiYgcoGyoBWcCJyJyAqtQiIioCxM4EZETOCYmEZFPsQqFiMif+CAPEZFPKRsagjOBExH5FBM4EZED+CQmEZFPZWwd+CkjB7sdAhGR5/gigQ/q39ftEIiIPMcXCdyOjtCJiJw0euhAy5fpiwROROR3544fYfkyfZHAhUVwIqJe/JHAWYlCRNSLLxI4ERH1xgROROSAgoNHLV8mEzgRkQPqmzssX6YvEjhvYhIR9eaLBE5ERL35IoE/fet5bodAROQ5vkjgp48ZhrsuP9XtMIiIPMUXCRwAjhvQz+0QiIhMs+NWnm8S+M1fHu92CEREnuKbBD5m+CC3QyAiMs2O1nS+SeAAkHvv5W6HQETkGb5K4KdlDcV/nD/O7TCIiDzBVwkcAO64ZKLbIRAReUJKCVxErhWRnSKyW0SmWBVUPJ8/+XgnVkNE5HmmE7iI9AXwDIDrAEwCcKuITLIqsHjWTrmyx9/XTBrjxGqJiEyzo1vsVErgFwDYrZQqVUp1AHgdwI3WhBXfySMG49U7LwQAPPdf52Pmf+dgQL8+uPizJ+CF/87BvJ9/FcMGhduN33rBBOT99goAwFWfM5/ohw1M3A59MMfuJCIdg/pbX2MtSpkb7F5Evg/gWqXUndrfPwJwoVLq7qj5JgOYDAATJkz4Unl5eWoRp2jdnnoMG9QPr6wrx3Vnn4RxIwZjUP++GD9qCEIhhc6QwoB+fVBw8Cgmjj4OQyIeIKpvakdZfTO+9JlR6OgMYVd1I460BDBu5GDsqWnCVZPGYFlhNcaNHIwXVpXiSGsAz/zgfAzs1weHWjqwt64ZNz23Do9+/4t4eEER5v70IlQebcXZpxyP9kAISwqq0N4ZwsB+fXDC0AH4bNYwHDzaihOHDcR3/7kWl52RhdFDB2DepgoAwN1f+ywOHmlF+aEWtHcG8ZOLJ2LN7nrkFlfjd9eehc+eOBQLtlWioS2Ac04ZgSED+mLcyMGobmjD4h1VOG/CSJw4bCC+etpoXPf3VTjcEsD5E0bgrLHDsbe2GWeNHYY/fXMS1u6pR0OfiGIAAAfASURBVGtHEJv2Hcby4hpMvuxUTFtUjNrGdgDABdmjMOGEIVAKeOymL+LN/P24790d6AyFj60fXfQZiABl9S1YtasWAHDGmKG49gtj8VRuCcYePwj/mTMe500YgQc+KED/vn2wu6YJIsDwQf1xwtAByBo6EGeMGYb3NlfgcycPx3njR+DsU47H2xsP4OQRg3GoqQOLC6oAAF897QScfuJQzFkXPta+MG44dlQ04DdXn4GbvzweF/41t3ufnnPK8fjPL4/HrqpGlNQ04eYvj8c9r2/B6KEDUdfUjhvOHosF2yvx7XNORnFVAy4/IwvjRw3B08t3o7axHUMG9EVLRzDmsXbFmVmobWzH8EH9MX7UYFww8QT89q2tveYbPqgfzjppODaUHYq5nBvOHovs0UNw1knD8cu5m3HLl8dj1a5a3HPV6fjdO9uPzffFsViwrRIXTByFUEihoS2AkUMGYPjg/mhsC+CT0tjLj+frZ52IU0YO7v4uI/3ia6fhmbw9uGDiKGzYe2zZo44bgEMRPe89dtM5+O1bW/GH68/Ca+v3oay+RXd9N+eMxxv5+3Xf7/q+n771PPxy7uYe711+RhZW7qrFt885GYWVDahrase4EYNRcLChe56u/annG58fgyUF1T2mTRo7HKdmHYf523p+bvigfhg+uD8OHG7VXV6k4oeuNT1Au4hsVErl9JpudwKPlJOTo/Lz802tj4goU+kl8FTK9BUAIh+PPEWbRkREDkglgX8K4HQRmSgiAwDcAuADa8IiIqJETPcQpZTqFJG7ASwB0BfALKVUgWWRERFRXCl18aeUWghgoUWxEBFREnz3JCYREYUxgRMR+RQTOBGRTzGBExH5lOkHeUytTKQWgNlHMUcDqLMwHDely7aky3YA6bMt6bIdALcl0meUUlnREx1N4KkQkfxYTyL5UbpsS7psB5A+25Iu2wFwW4xgFQoRkU8xgRMR+ZSfEvhMtwOwULpsS7psB5A+25Iu2wFwWxLyTR04ERH15KcSOBERRWACJyLyKV8kcDcGT06GiIwXkTwRKRSRAhG5R5s+SkSWikiJ9v9IbbqIyFPa9mwTkfMjlnWbNn+JiNzm0vb0FZHNIjJf+3uiiKzX4n1D6z4YIjJQ+3u39n52xDJ+r03fKSLfcGk7RojI2yJSLCJFIvIVH++TX2vH1g4RmSsig/ywX0RklojUiMiOiGmW7QMR+ZKIbNc+85SIWD/wZPxteVQ7vraJyLsiMiLivZjftV4+09ufcSmlPP0P4a5q9wA4FcAAAFsBTHI7rqgYxwI4X3s9DMAuhAd6ngFgijZ9CoDp2uvrASwCIAAuArBemz4KQKn2/0jt9UgXtuc3AF4DMF/7+00At2ivnwPwM+31zwE8p72+BcAb2utJ2n4aCGCitv/6urAdcwDcqb0eAGCEH/cJgHEA9gIYHLE/bvfDfgFwGYDzAeyImGbZPgCwQZtXtM9e5/C2XAOgn/Z6esS2xPyuESef6e3PuDE5eSCa/NK+AmBJxN+/B/B7t+NKEPP7AK4GsBPAWG3aWAA7tdfPA7g1Yv6d2vu3Ang+YnqP+RyK/RQAuQCuBDBf+2HURRyk3fsD4b7gv6K97qfNJ9H7KHI+B7fjeISTnkRN9+M+GQdgv5bA+mn75Rt+2S8AsqOSniX7QHuvOGJ6j/mc2Jao974L4FXtdczvGjr5LN7vLN4/P1ShdB28XQ5o0zxJu1w9D8B6AGOUUl0joVYBGKO91tsmL2zrkwD+P4CQ9vcJAI4opTpjxNQdr/b+UW1+L2zHRAC1AF7SqoP+JSLHwYf7RClVAeAxAPsAVCL8PW+EP/cLYN0+GKe9jp7ulp8gfBUAJL8t8X5nuvyQwH1DRIYCeAfAr5RSDZHvqfBp1dNtNkXkmwBqlFIb3Y7FAv0Qvtx9Vil1HoBmhC/Xu/lhnwCAVkd8I8InpZMBHAfgWleDsohf9kEiInIfgE4Arzq5Xj8kcF8Mniwi/RFO3q8qpeZpk6tFZKz2/lgANdp0vW1ye1svBvBtESkD8DrC1Sh/BzBCRLpGb4qMqTte7f3jAdTD/e0AwiWYA0qp9drfbyOc0P22TwDgKgB7lVK1SqkAgHkI7ys/7hfAun1Qob2Onu4oEbkdwDcB/FA7IQHJb0s99PenPrvrvyyoc+qH8E2LiThW6f95t+OKilEAvAzgyajpj6LnzZoZ2usb0PNmzQZt+iiE621Hav/2Ahjl0jZdgWM3Md9Cz5srP9de/wI9b5a9qb3+PHrewCmFOzcxVwM4U3s9VdsfvtsnAC4EUABgiBbfHAC/9Mt+Qe86cMv2AXrfxLze4W25FkAhgKyo+WJ+14iTz/T2Z9x4nDwQU/jSrke4ZcceAPe5HU+M+C5B+DJwG4At2r/rEa7XygVQAmBZxEEnAJ7Rtmc7gJyIZf0EwG7t349d3KYrcCyBn6r9UHZrB9lAbfog7e/d2vunRnz+Pm37dsLGlgEJtuFcAPnafnlP+/H7cp8AeBBAMYAdAF7REoPn9wuAuQjX2wcQviq6w8p9ACBH+072APgHom5aO7AtuxGu0+763T+X6LuGTj7T25/x/vFReiIin/JDHTgREcXABE5E5FNM4EREPsUETkTkU0zgREQ+xQRORORTTOBERD71fzZZUL/jMqnrAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(losses_ru)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "25 epochs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x182b86970>]"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAD4CAYAAAD1jb0+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deZRcZZ038O9PQFTQIQw5vIiciSIz78HXIWKLjiCDggLBOcg4zivDMKijOAhz1MFBJI7AIAQIAYadQEIChh0CIRvZOvva6SS9Zel0p7uTTm/pvdNrVT/zR91Kqqtructz1/p+zslJ9a1b9/5uLb/73Oc+iyilQERE4fMRvwMgIiJ7mMCJiEKKCZyIKKSYwImIQooJnIgopE70cmdnnHGGmjRpkpe7JCIKve3btx9RSk1MX+5pAp80aRJKSkq83CURUeiJSH2m5axCISIKKSZwIqKQYgInIgopJnAiopBiAiciCikmcCKikGICJyIKKSZwogLSNxTDjoZOz/a3dl8bDnb0Z3yu8+gwGtozP5eud3AEe5p7TK3b0N6P/a19pmNM2tvci5aeQcuvS1XR2I3DXQOOtmEFEzhRBCilsLspf4L7xbxSXPfMRvQNxWztJxYfRXzU/BwC/zJ7Ky6dXpzxuUsfLs76XLobZ23FVY+vM7XupdOLccWja0zHmHTl42vx1QdWWn5dqu8+uR5ff3CVo21YwQRO5LJDnf3oH7aXMM1asOswrv6fdVha0ZxzvbJDXQCAkdiorf18fuoSXPv0ekuvyTZnTK+Fk8jOg12W9lkomMCJXHbJQ8X45xe3uLqPPc29AICaNutVB1ZVNJqryiD3MYETeaC0gSVI0o8JnIgopJjAiRxo6h5Ao+ZWB6v3tmLSnYvQeXRY63YpepjAiRz4m2mrcLHmVgcz19YCAKpMtCqhwsYETkQUUkzgREQhxQRORBRSTOBERCHFBE5EFFJM4BRqA8NxbK/v8DsMIl8wgVOo/efbu/D9ZzehudvZKHJEYcQETqFWdTjRVvqoy4NFBZ34HQD5ggmcKGCyjd5HlC5vAheRc0SkWESqRKRSRH5pLL9HRBpFZKfxb4r74RIVDpaqKZ8TTawTA3C7UqpURD4JYLuILDeee0wp9Yh74RERUTZ5E7hSqglAk/G4V0R2Azjb7cCIiCg3S3XgIjIJwJcAJEenv01EykRktohMyPKam0WkRERK2traHAVLRETHmU7gInIqgHcA/Eop1QPgWQDnApiMRAl9RqbXKaVmKqWKlFJFEydO1BAyEREBJhO4iJyERPKep5R6FwCUUi1KqbhSahTACwAuci9MIiJKZ6YVigCYBWC3UurRlOVnpax2HYAK/eEREVE2ZlqhXAzgRgDlIrLTWHYXgOtFZDIABaAOwM9diZDIBLNtp4dicbT3DePTp33c3YCIPGCmFcp6ZG6Sulh/OEQWWWws/e+v7sCyqhbUPjAFH/lIMFtaK7AnD5nDnphUUJbvbvE7BPNsnF8Uu3EWFCZwogiQYF5MkMuYwClQlFK4b2EVKg93+x0KUeAxgVOg9AzEMGv9AVw/c7PfoRAFHhM4EVFIMYETEYUUEzi5YigWx6HOfr/DGIeNNChKmMDJFbe/uQuXPFSMwZG436FkxFYbFAVM4OSK4j2tAIDYqL0ir/VXsWhNhYcJnILFYsk4igVpVvOQWUzgRAElkTw9kU5M4EREIcUEThRQHNSK8mECp2ByOXdJgJuhOAmN9eeFhQmcAiXAeTXQWF9emJjAiYhCigmciCikmMAd6h4YwdyNdRxIPwu+L0TuMTMnJuVw1/xyLCprwhc+/SkUTTrd73ACw6ubhDw9UCFjCdyhzqPDAICh2KjPkURLISdmXrSQWUzgFChWy+1RbnvBliWUDxM45dXWO4Tu/hG/wyCiNKwDp7y+cv8KnHSCoPr+KX6HQkQpWAInU0birJglChomcAqMmrY+xIwTBZsfEuXHKhSyZWPNEVzwmdNwysl6vkJtvUO4fMYa/N0Fn9ayvULF015hYQmcLDvcNYB/emELfvPWLm3b7BlM3CTdUtuubZuFhGPIFKa8CVxEzhGRYhGpEpFKEfmlsfx0EVkuItXG/xPcDze4CumKv384BgDY19LrcyREhc1MCTwG4Hal1PkAvgbgVhE5H8CdAFYqpc4DsNL4u+Cw5JNbAZ3XtOF7lh3vjYyVN4ErpZqUUqXG414AuwGcDeBaAHON1eYC+J5bQbqltKET1SxFusLr81oUf9csHFA+lurARWQSgC8B2ALgTKVUk/FUM4Azs7zmZhEpEZGStrY2B6Hq9/fPbMS3H1vry76/Pm0l7nhbXx0yAFw2vRjTFu/Wuk2/mM3HQZ6YgchtphO4iJwK4B0Av1JK9aQ+pxLXNRl/c0qpmUqpIqVU0cSJEx0FGyWHuwfxZskhrdusa+/H82trtW6TiILLVAIXkZOQSN7zlFLvGotbROQs4/mzALS6E6I+t/xpO373bpnWbUbx0t0PLEcTWWemFYoAmAVgt1Lq0ZSnFgC4yXh8E4D39Yen15KKZry29aAr2+aVPBF5zUwJ/GIANwL4lojsNP5NAfAggG+LSDWAK4y/ySMHO/rx2tYGv8MgIh+ZaYWyXiklSqm/VkpNNv4tVkq1K6UuV0qdp5S6QinV4UXAbmjsGvA7hIz+ZfZWnDd1ccbnfvDcJvzu3XIMxeKmtrXzYBeueWIdBobzr7+0oglFf1yO4SxjnHtRbcSqKaL82BMTwN89ud7vEDJau68t6yBSHcZEEmbdt7AKlYd7UHm4O++6935QhSN9wzjSN5RzPbYAIfIXEzisJ8NMWGLMTSmFJeVNiMXNzVzEc4M9+b6H/J664+hQDK29g57vlwncISaa3JIJY3F5M26ZV5q3mSPzC2y9CZZnMuL3VqspT6zDRfev9Hy/oUvg//VeBbYe8K+6/ZEP92LSnYswOspUk1Nagmg/mqiOae7OXEpxWh0TxU+DOTY86tv7fdlv6BL4K5vr8Y/Pb/Jt/8+s3g8gmgkjjJjkqJCFLoEHyd7mXmzYz+FP3cC6WsqE34uxIpfA644cRVtv7tYTulz5uD/jqERZIZSo+4djmLPhAEfWI8ciNyPPZY+sBgDUPXiNv4EETK5kMTqqMBiL4xMfHft1YHpxx7TFe/DK5np8+rSP4ztf+D9+h0MhFrkSOOWW6V7h4yurcf4fPkR3f2JWHD9LwYVw0ugaSLzPAyO5O1UVwntBzjCBExbsbAQAdPQ7bw9vVyFUnZjGN4NMinwC39HQid+8tctRfeO+ll7UHTmqMapoqmnrwxMrq7VsS7H8SZRXZBP4q1sSAz396KVteHv7IXQbl625bNh/BPEM7bu/89jaY3XrSV7egPrZyyWe7cuJH87cjEeX7ztWFWNHtsJnfFTlfM+/+8R63LewyvZ+A4XnLjIpsgn8rvnlltbfsP8IbnhxC54u3p9zPT/G/1he1WL7tWa7ruswlKlOV1MyOveuxTlPZMPxUcxaf0DPzgKCNSmUT2QTuFUtPYkeggciVFVSebgbn5+6BKv22D8B2OVG8lmxO/BzhhB5igk8j/p27xJ6TVsf+odj2rZX2tAFAFgZoMSXr26btQfO8N5BYWECz6F7YAR/O321J/saHVW4fMYa/PyV7a5s387POr3Oed6WehwdsneCyVci5+BKDvENLEihTeAj8dGMNxx1spus7EgeyYb9R1zeU/4ferZ6/qnzK3DvB5W6A8qInRSJ8gttAj9v6hLc8OJm17Z/2KNZevJNmhA0HUfttzAhIr1Cm8ABYHOtO8PKfrDrML7+4CpsrMk+UJWuAmK24VXNMlNS1V2YZeGY/MLv3lihTuBu2WHc/Nvd1DPuOd01je12ZwMyEYjbtaJ2t9/aO4jatj6tsRAVIiZwTeze/b9p9ta07UTfRfevxLdmrPE7DKLQi9xohGEVnjYE5k4xybXueGcXzpnwCfzFn3/C3OuUlb1EE5sCklksgRcIO606dLQE+bCyBS+a6CEpITqFecWPXr+kV3VLr6stywqmBM7ZuhO8yAkKiuk4B07kUDi+/Vhi0he35ieIfAncasKKYuLxKl9EpcQYi4+aGvzMqai8X+Sf0Cfwn84twQoHgz3lkivxuZUUdW63EPKDG8f46zd34YJ7l+nfcJru/mH8dO42dNhtiUQFL/QJfMXuFvxU83CruZJCzOXen2GS/k6YqRoIQ+3BB7sOe7KfuZvqsWJ3K+Zs0DeKYhjeX9InbwIXkdki0ioiFSnL7hGRRhHZafyb4m6YwReWm3BOf+BmXp/xvTBdVGYzFDvC8e0j3cyUwOcAuCrD8seUUpONf4v1huW90YgXXZL5c1PNEfzbK9u13EizlDTy7M/N6p7u/hFM/u9lKKlzp+duNnfNL/ds7Bg/PL+mBk+t0jMDE9mTN4ErpdYC8Pab74M3Sw5aWj992Nf/eHMnhmLjJzSoaetDa6+z7vLOHU+ejyzbh6WVzRiKeTfRQyo/6uVLD3aiq38ET67KPVmHbq9uacBLG+o83Wcmw7HEZBex+OixAdqGNXz+05bswSPL9jneDtnnpA78NhEpM6pYJmRbSURuFpESESlpa2tzsLv8dh3ssv3atl5rg0pN/u/lY/5u7R3Cun3j23tePmMNLrp/pe24nEitykgvAIdlmrZCpPti8MX1tbhvYRXmbWk4dg/njW3WCiwUTHYT+LMAzgUwGUATgBnZVlRKzVRKFSmliiZOnGhzd+Zsrs0++JQdYeoRl5xRyKx11dk7FxRC65UgcLs9eLJQ0juYKHX3pQyPbOZm/J7mHmzR/Jtyim3ox7KVwJVSLUqpuFJqFMALAC7SG5a/dOWvUZMtVoZicVRlGDjLjOQl8TVPrLP1+lzcaHHT2mPuSidMJ0/Hspwx0xffNHsr7llgrk59SXkTvnL/CmzKMaJmPlc9vg7/f6Z7Qzbr5uX8r0FhK4GLyFkpf14HoCLbuoWqZ3AEe5p7Ta37+/kV+O6T623t5/oXEj+wI3362xJ/85HV45Y5TavL8rTZZwEruzX72jBnY52pdbfVdQJIzItaCJZWNOHzU5dgr8nfXFTk7UovIq8BuAzAGSJyCMDdAC4TkclI/J7rAPzcxRg9oTtvXPnYWjSZHOu7tKHT9n72tQRzWFa7vQydfg5zNhzAkb5h/ObKv3K4pegohJNismBQ3lgYJ6ykvAlcKXV9hsWzXIhFu65+/3q4mU3eAFDT5u7EyWH8AY/EFX7w3EbMvLEIE075aNb10tuc3/NBFQCEKoF39Q+jpWcQZ37qYwCAknr7J/RceG8jekLfEzOXW/5UeuyxvUl99cXih3tcaoOcd2AwTdcz2+o6MX9Ho5ZtBVHyXXp5Uz2++oCzlkoh/6qSTZFO4Ie6+j3tofZhZbOHe8sv2da3tu2o7Tk+9zRnv7maXqIL2+BMgyNxvLypbszN5ob2fv8C0iD5EYTsoyCbIpXAdZRC+ofHd8Yx663thzREoN/Wug60Wmjnnvrj/7dXtrsQUTA8tnwf/vB+JRaVNx1bdun0Ytvbi8VH8dDSPZ6MZJhN2K8anSq0ZoaRSeD17XrqkZdWNofictSvApaZH0jPQGxcEjM796crx5Ul5E7jHkl6r1q7FpU34dnVNXhwyW5T4dRqvPeR630rhCaZYRmLSLdQJfBcM1v87fTV2vYThpO4GyHWtPZh2uLdjo//6eLxXdYfX2FuzIxxIxzmWHf13lbsbTHfbMztaoWReCJav4YpyCfKSa4QTlKZhCqB3/DilpzPF+9pNb2tv5y6ZMzfmZJW8V7z24uCn75cgufX1lpqQZOJVz+mF9fpG4Y1rJRSeHJlNdqPZq4iy3cy3lzbjuue2aBlbBSnFpYdRoXDZoBhuw/jVKSmVNtyIPuYW3tSejoqpTCco9dWMgHV+3RDazg2ivk7/KtPT/0JWOmNGfT6x4CHZ8v2+k7MWD5+QCmzaey375Shvr0fh7sGMOmMU/QGZ9Ftr+4A4N70Y1EUqhK4VamXjKk3J1PHhAii+KjC3I31OdfxqsRkJ+m5eanePTCC+xZWYX9rLwZHTN5w1hzOwY5+/P69csQDMLnHzLW1+VfyP0zPBL0QoVukSuC6uF1XqOummQ5KKcsjMeZTZuEyeEl5Ey4+74xjf+ebXuyrD6zA4Mgo3iw5eGyQJrvs/tZ//cZOlNR34trJZ2fZsP2YrDjY0Z93aIJUUahdyPbWRrl+P5dIl8BTWZl2za0venvfECbduQjzNjdo3W6rxZEIU728qR5H05pOmq02yZYAewfNNaM7cOQobplXiv94Y6ep9QFgcGTU+D936fu+hVU5n39m9f5jzT6z/fhj8VEc6hxfjZY87PRXeZ1CzHxO+e5HFFiBNXIKJoGnyvSd1XXjLdeN1OrWxLglr2zOXT1i1eUz1th+7focLXvcNmCcOA51DmBtdeax4u1eEs9afyDna82cRP+4aDcueahY+xWKF6wWQtLXr2mzNsbO4a4BLElpT0/eKMgEnomuksiP52zTs6EcHl66Z8zfvT7V6WdLEnauYKbO1z+gpdNJC5InFT875uiQ6aud7zNKLRSs2tOSt5fxdc9swC3zSnOu4yY2I4ygbF9Svy4b75pfbmq9fOODP7O6RldIeQW13tTMZ7jJp8kIFpY1mb/BaoPduT3tfpQ/mVOCn+fpkdticpx30isUCbzz6DAm3bnI1mvttAt1K8Gb7XmXqx222UkidLHy9iXf66DWq67ZZ3FKP5vHMRwfxf2LcvfGdOKRZXsBmEvIuT6Lp1ZVo6Ej3GO/JPEmZoBZadVgSkATjBlv+zzeSvpbd7Cj3/Gwvfkmvpi51t4Vx7HkleHzbkwZ3OuOd8oy9h5N9qhMPYkNjsSxPctwr6nrOe0MZdexG6wmzrypExI7SYBeFyrouFAkcK+N+lyE/NFLW8ctS95Ia+vTd6m6vb4j76X+wY7coxh+4+FifCvtJqrZUrvZessHFu8Zt0z3JzT9w73jljVmGMHx+TUm2l3nYTZVhqVM2eygFRQ5U5AJ/GCGpmGplrk8LGy+xJWpPvEr96/QHsf3n92Uc3LjbBaWHR7zd76221EyGHOvbtstmVrjhLXKobHT3rDIVrRrLCS5LfIJPNPXNNP8k6nf8cM+Xf7mM2Jy0tYhF5NMa88Qnlw1vrohPqqOtdbIV2rXwUwvSIVElcfGGnMnqcVZmsGZSXWdATuJVRuDfKX2QK7LMTTEpdOLMTAcR21bn+37TTqtrz6SMZFelmGe1lQ6rsy+/Ef9hSW3RDqBh7OMkd1v3y4ztV6uURudytY65p1S63XzXpQC71lQiRdMDnr1iyzN4MzUJ2eq9nJbrrCK9yZu2G5NGR/ota252753D4yMqw7zw+iowj/P2oJ/eiH34HUU8QQ+MBJ33N3ajmkZxoPuSemdaLeKfbnJbtM/mVPiSdvl1A4u5Yes32ie8sQ6neFktKOhy7Vtp36MVoa1DcItv6A2DwWOvz/VrdZnmA/wYbki0gm8pWco56iDVq3L0lswXaYbXXe8dbz0fMiDerwL7l3m+j46+4+fJHT3Lk1yej/ZSmJ1yq2k2DcUczRIk1/35FMLHDsa3JmoudCFIoF7cVbN1OIg3Y2z7F8mt/Q6r1cPQsktTHR17c72/Usd1TI5Roturb2D+H93f2i785aVzkw6ejMmTzQVjd34Wcr4Q9c9s9Hxtmk8jkZIkWVlLPNcUkvWqcn8xy+5P2xCs3FDfWlFsCbMzser4Qdq2vpw6snH01ihFXKYwIksCFKCCGtTQJ2SY7b8w5c/43Mk/ghFFUoUuHkzLcruX+xel3Tdfv3GrrzrNHcPHmvil4/I8frrct29kTPIV1d+9/v6BxzT7Q6TLbWiggk8RII+k1BUCQQXP7gK//We8wT2tWkrsa/F2lCt6UYV8NKGA+gfMfd90HXVMHdT/hvVyZOA1zdOrexveVULai0OlwsADe39uPv9ikDMxJSUN4GLyGwRaRWRipRlp4vIchGpNv6f4G6YRP5q7BpwraWNVVsPdODeD6rw4JLxQwyYVYiVL72DIxgcieNnL5fYau/+76+VYu6mek+uhswyUwKfA+CqtGV3AliplDoPwErjb6JIstM8cMVu81Od2dWT6UahyaJokNuB6/LGtrEdl754zzJ8M09PzlxG4on31s5b193vzk3dvAlcKbUWQPoAxNcCmGs8ngvge5rjGqMQvmxEmXh5sa5jX8cHgBy/tWlLdltqz24lnkw54rfvJMbfv3VeKRaVJZqU2h0lclFZU84x+vOx01PZDLt14GcqpZKNbJsBnJltRRG5WURKRKSkrc3ieMw0TqZR88hd//Dc8TbMTsoSDTnGInGTF/XRZgpZz6+pNdWxSnd5bVF5E2591dlsQUtdHuDOLsc3MVXilJr1K6KUmqmUKlJKFU2cONHWPoJ004AKj67ZZi6dXqxlOzq87nC6uXS8SPaH3QTeIiJnAYDxf/aZfDV4aClLnRQMXhclnHShz8Wt4RzsxLutrgP7W521zHl/Z6Oj11tR125uZi0v2E3gCwDcZDy+CcD7esLJbLeDuieisBgYjqPskPn+AnamC8zFyckiPRYrm/rBc5twxaPORkFM3mD0wi9f32n5NW5Fl7cnpoi8BuAyAGeIyCEAdwN4EMCbIvKvAOoB/KNL8REVjNvf2onF5cGsa9WFvUf1ypvAlVLXZ3nqcs2xEBW09ORd0diDL37mzxxv1+wgVa9vNV8vnmkO0SiJjyp8RPRf5ejGnphEFnj5c757QaWHewOespCUs7WGcqOqwI9GDOfetRi3v5V/aAS/MYETkSMVjd1ak2z6ltY7mGHqvoVVtl/7bql3N0btYgInCjC/JmOw4uZXtpte106NxKiDN2HWenPT6bnNrdZETOBEEeflSUDHvpp7BrVty6nBEfcmCNeBCZzIggDklJz8jk/HrD6Xz1jtPBBNHl2+z+8QcmICJwowK+3CszEzXaAuf9rcAKWyp3FBolSbK0m7NT2dHR1Hh/0OIScmcKIAKzvkfOhSL6sB7ltYhR0Hs590FIC9zb2oaQtOb8ZclEoMQ/vBrsN+h5IREziRBUFqFRykWFINx/SWoN06zviowkg8f6zTHIy77jYmcKKQ8ru+O5sg3Hw04/oXNuO8qUtyrqOgMDAc3BuZTOBEFlQc9nZcHh2zv/iRULM1mwvSVcPWA+nTHIQPEziRBWv3eTumfa5R+jIlww8rm7G9vtO9gDz27Ooa/Oilbf4FEPCrCSZwopDK1ClmX0sfvv/sxvFPeEhHU8Kkh5YGs/5ZKYWnVlXjYEfmSTp+MmfsSWdI832BJCZwopAKal1zbdvRnLEFNOyMFID6DON/1x45ikeW7cM3Hi7G/6yoHvf8qj1jp0iIuTTcbd7RCIkomEoCWlXy+/cqcNFnT8/4XMAH98uotGF8s8jUsV8eW5G5s49b3edTsQRORNrtaMh+cnGaww929CNmovmfDtmScKmJk+efNtfrDmccJnAiCpVvPFyMh5buQaePvSRrj+TviLRyj6szTQJgFQoReUpPHcrGmna8sM6/kQa9qB4xgyVwoojT2Sqk0Dh557zI8UzgROShYJ5ONlicNMJMcj7SN3TscbYbnU4xgRORdtlmib/i0bV4b4fzmW4qNfeIveHFLRmXZ0vUZk5CumPMhAmciDy1IKAj+1kRkCpwJnCiqAtKskkK+hjbqQL21o3DBE4UcWaGTCVrglKTzwROFHHb6oLZYzMMsjUXDMpVDRM4EZFFbAdORBRwqWOepKprzzwKodeYwImIslhS0ZxxeTDK3w670otIHYBeAHEAMaVUkY6giIiCbMSl8b2t0jEWyjeVUta6MRERBUT/cMzyazbVtrsQiXWsQiGignb+Hz70OwTbnCZwBWCZiGwXkZszrSAiN4tIiYiUtLV5O58gEVGUOU3glyilLgRwNYBbReTS9BWUUjOVUkVKqaKJEyc63B0RESU5SuBKqUbj/1YA8wFcpCMoIiLKz3YCF5FTROSTyccAvgOgQldgRESUm5NWKGcCmC+JWUpPBPCqUmqplqiIiCgv2wlcKVUL4AKNsRARkQVsRkhEFFJM4EREHnBjACwmcCIiD+xr6dO+TSZwIiIPuDEJBBM4EZEHhkb0D4DFBE5E5IGyQ13at8kETkTkgSxzQzjCBE5E5AG2QiEiCimWwImIQsqNadiYwImIPMAqFCIiOoYJnIjIA6MsgRMRhZML+ZsJnIjIC7yJSUQUUiyBExGFFAezIiIKKZbAiYhCiu3AiYhCiiVwIqKQ4lgoREQhxZuYREQhxSoUIqKQYkceIqKQYisUIqKQYhUKEVFIBW40QhG5SkT2ish+EblTV1BERFETqDpwETkBwNMArgZwPoDrReR8XYEREUXJsspm7dt0UgK/CMB+pVStUmoYwOsArtUTFhFRtHQPxLRv00kCPxvAwZS/DxnLxhCRm0WkRERK2trabO3omRsutBchEVFArLvjm9q3eaL2LaZRSs0EMBMAioqKbFUDTfniWah78BqtcRERhZ2TEngjgHNS/v6MsYyIiDzgJIFvA3CeiHxWRD4K4IcAFugJi4iI8rFdhaKUionIbQA+BHACgNlKqUptkRERUU6O6sCVUosBLNYUCxERWcCemEREIcUETkQUUkzgREQhxQRORBRS4sYYtVl3JtIGoN7my88AcERjOH6KyrFE5TiA6BxLVI4D4LGk+gul1MT0hZ4mcCdEpEQpVeR3HDpE5ViichxAdI4lKscB8FjMYBUKEVFIMYETEYVUmBL4TL8D0CgqxxKV4wCicyxROQ6Ax5JXaOrAiYhorDCVwImIKAUTOBFRSIUigYdh8mQRqRORchHZKSIlxrLTRWS5iFQb/08wlouIPGEcT5mIXJiynZuM9atF5CaPYp8tIq0iUpGyTFvsIvJl473Zb7xWPDyOe0Sk0fhcdorIlJTnfmfEtFdErkxZnvH7ZgydvMVY/oYxjLIbx3GOiBSLSJWIVIrIL43lYfxMsh1LGD+Xj4nIVhHZZRzLvbn2LyInG3/vN56fZPcYs1JKBfofEkPV1gD4HICPAtgF4Hy/48oQZx2AM9KWPQzgTkxF6Z4AAAOvSURBVOPxnQAeMh5PAbAEgAD4GoAtxvLTAdQa/08wHk/wIPZLAVwIoMKN2AFsNdYV47VXe3gc9wD4TYZ1zze+SycD+KzxHTsh1/cNwJsAfmg8fg7ALS4dx1kALjQefxLAPiPeMH4m2Y4ljJ+LADjVeHwSgC3Ge5hx/wB+AeA54/EPAbxh9xiz/QtDCTzMkydfC2Cu8XgugO+lLH9ZJWwGcJqInAXgSgDLlVIdSqlOAMsBXOV2kEqptQA63IjdeO5TSqnNKvHtfTllW14cRzbXAnhdKTWklDoAYD8S37WM3zejhPotAG8br099T7RSSjUppUqNx70AdiMx32wYP5Nsx5JNkD8XpZTqM/48yfincuw/9fN6G8DlRryWjjFXTGFI4KYmTw4ABWCZiGwXkZuNZWcqpZqMx80AzjQeZzumIB2rrtjPNh6nL/fSbUbVwuxktQOsH8efA+hSSsXSlrvKuOz+EhKlvVB/JmnHAoTwcxGRE0RkJ4BWJE6INTn2fyxm4/luI15tv/8wJPCwuEQpdSGAqwHcKiKXpj5plHRC2WYzzLEDeBbAuQAmA2gCMMPfcMwTkVMBvAPgV0qpntTnwvaZZDiWUH4uSqm4UmoyEnMAXwTg//oZTxgSeCgmT1ZKNRr/twKYj8SH22JcrsL4v9VYPdsxBelYdcXeaDxOX+4JpVSL8aMbBfACEp8LYP042pGomjgxbbkrROQkJBLePKXUu8biUH4mmY4lrJ9LklKqC0AxgL/Jsf9jMRvP/5kRr77fvxuV/Tr/ITHtWy0Slf3Jiv0v+B1XWoynAPhkyuONSNRdT8fYm04PG4+vwdibTluN5acDOIDEDacJxuPTPTqGSRh7809b7Bh/w2yKh8dxVsrjXyNR9wgAX8DYG0m1SNxEyvp9A/AWxt6s+oVLxyBI1Es/nrY8dJ9JjmMJ4+cyEcBpxuOPA1gH4LvZ9g/gVoy9ifmm3WPMGpNbPyTNb9wUJO5e1wCY6nc8GeL7nPFm7wJQmYwRifqulQCqAaxI+fEIgKeN4ykHUJSyrZ8gcVNjP4AfexT/a0hcxo4gUe/2rzpjB1AEoMJ4zVMwegB7dByvGHGWAViQljimGjHtRUorjGzfN+Nz3moc31sATnbpOC5BonqkDMBO49+UkH4m2Y4ljJ/LXwPYYcRcAeAPufYP4GPG3/uN5z9n9xiz/WNXeiKikApDHTgREWXABE5EFFJM4EREIcUETkQUUkzgREQhxQRORBRSTOBERCH1v/GOz0pTpgICAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(losses_ru)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "60 epochs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x3925189a0>]"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAD4CAYAAAD1jb0+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXQc1Z0v8O8vbMmDTIBBh/iQvBHMZEjgZTBEQ9aXl5CQAZIzkLxMTkhehplh4kzizCMveS8RIYEQdgImkACxwYAhGNsxi4nlTV7AC7ZsWZZlWbJsWZYXWZbkRZa8aL/vj66We+/q6ltV91Z/P+fI7q6urvp1Lb+6devWLVFKgYiI7POusAMgIiJvmMCJiCzFBE5EZCkmcCIiSzGBExFZ6vQgZ3bBBReo8vLyIGdJRGS9jRs3HlRKlaUODzSBl5eXo7a2NshZEhFZT0R2ZxrOKhQiIksxgRMRWYoJnIjIUkzgRESWYgInIrJU3gQuIu8WkfUisllEtorI3c7wF0Rkl4jUO38T/Q+XiIji3DQjHARwjVLqmIicAWC1iCx0Pvt/Sqm5/oVHRETZ5C2Bq5hjztsznD/2QesYGR3DnA17MTrGRUJEwXJVBy4ip4lIPYBuANVKqRrno/tEpEFEHhORs7J8d5KI1IpIbU9Pj6awzfHi2t346asNeLkmYzt7IiLfuErgSqlRpdREAB8AcLWI/DcAtwP4MIC/B3A+gJ9l+e40pVSFUqqirCztTlDrHTkxBADoPTEcciREVGoKaoWilOoFsALAdUqpTqd6ZRDA8wCu9iNAIiLKzE0rlDIROdd5/R4A1wLYJiITnGEC4CYAjX4GSkREydy0QpkAYIaInIZYwp+jlJovIstFpAyAAKgH8B8+xklERCnyJnClVAOAKzMMv8aXiIiIyBXeiUlEZCkmcCIiSzGBExFZigmciMhSTOBERJZiAicishQTOBGRpZjAiYgsxQSuiWJvskQUMCbwIknYARBRyWICJyKyFBM4EZGlmMCJiCzFBE5EZCkmcCIiSzGBExFZigmciMhSTOBERJZiAicispSbp9K/W0TWi8hmEdkqInc7wy8WkRoRaRWR2SJypv/hEhFRnJsS+CCAa5RSVwCYCOA6EfkEgIcAPKaU+hsARwDc6l+YRESUKm8CVzHHnLdnOH8KwDUA5jrDZwC4yZcIiYgoI1d14CJymojUA+gGUA1gJ4BepdSIM8o+ABdl+e4kEakVkdqenh4dMRtJgd0RElGwXCVwpdSoUmoigA8AuBrAh93OQCk1TSlVoZSqKCsr8ximwYT9ERJROApqhaKU6gWwAsAnAZwrIqc7H30AQIfm2IiIKAc3rVDKRORc5/V7AFwLoBmxRP51Z7RbAMzzK0giIkp3ev5RMAHADBE5DbGEP0cpNV9EmgDMEpF7AWwCMN3HOImIKEXeBK6UagBwZYbhbYjVhxMRUQh4JyYRkaWYwImILMUETkRkKSZwIiJLMYETEVmKCZyIyFJM4ESaKaUwp3YvTg6NZh1nbExhe1c/TgyN4PjgSNJnI6Nj6Dx6MuP3+geG0dZzLONnJlna1IX1uw4HPt+xMYWxsdLpl8iKBD6/YT/KK6vSNvS4//GbFbjmkbcAAPPqO1BeWYUTQ6fG3d7Vj/LKKrR2p2/4H79/KR5etC1t+JW/XoI/rtudNnzir5fgIWf8ObV78cSyHUmff/vZdZg8sy7te5Nn1qG8sipp2AMLmvHpB5enjXvbrE347ou1ScOOHB9CeWUVyiurxucfd/eft+LrT7+TNp3ZG/bg8jsXYTRhg56ypCUtjkS/W7YDH7pjwfj7b05bi689tQYA8OiSFlw75e3xz/5UuxfllVUYGhkbH/a1p9ag8tWGpGn+eHY9rr5vacb5DQyPoryyCm9sivXE8Ld3LMT01bvSxvs/s+tx+Z2Lkoat2NaN8soqHDo2mDb+0qYulFdW4cjxIfz9fUvx7Kq2tHGuuqcat83ahKMnh1FeWYW3WrrHPxsaGUN5ZRVmb9iT9J2Ke5fi/gXNGX9L3JrWQ/jp3AZ85M5F+J8Z1gsAPLmiFV96bCUuu3MxLr9rcdJnd725FZ98YDmOHB9K+963nqnBNY++nTY87qN3LcaTK1px3W9X4t75TWmf3zO/CVfdU5007LHq7fjE/cvSxp30Yi0uuf3UttLYcRTllVVoP3h8fNhnHlqOKUta0r777y/W4htT1wIAKu6txn1Vp2L51ZtbcePvV2f9DQBw57xGlFdW4cYn1yQN/89XNmXc1uM+/MtF+MKU9OXT0z+I8soq/O0dC/GLN7YkfTYwPIpLbq/CvPrk3kB+Mmczbn1hw/j7sTGFK+5egjkb9mad/0fvWozyyir8/PUtWcfRyYoE/vjSWJLs6M1cKtl96ATanI3qcSeh7u8dGP88vmIWNXamfberbxBPvbUzbfiRE8P4xRuNacN7TwzjaWf8eFwAoJwcuab1EKoa0ueTadjUlW0Zf9O8+v2obupKGrYj4eDzdEq8z69pR+3uI2nTuXPeVhwfGsXw6KkE+8Ty1rTxEj1avR3Do6cS/rq2w6jb0wsA+N3y1qQ44geSoyeHx4fV7enFrJQN/LVNHejuT0+yANDVF1tPU6q3AwCGRsdwT4bE8/qmDhxPKdE+uzqWlJs7+9PGn+Yk7JaufvT0D+LeqvSke/j4EObV70dzZx8AJG0H8d/0m8XJyengsUFMW5l+MEh0bPDU8tiYYb0AQP3e3ozDm/b34eWa2EGjfyC9wLKl42jOefcPjuA3i1uw7UA/ns1wIJy+ehcOpxwYHl+2Awf6BtLGXdLUhcTC7Gt1sf1oafOpbXPfkZN5t6mDx4bwzKpTsbzwTjs278v9O15cGys8bU5ZTn/evD/jth43NDqGXQkHmLja9sPjn/9xXfJBuad/EGMqfV2/WrcPy7YlHNRHx3D05DB+OS89L8T1O4XMmTV7so6jkxUJnKhUbNqbPTkRpWICJyKyFBM4EZGlmMCJiCzFBE5EZCkmcKI4Tc2HlYvp8El8pAMTOJU85lKyFRM4EUVeVM94mMCJiCxlVQJ3U7dIpSOK24MkVOhEtdRI+liVwIkyYaKjUpU3gYvIB0VkhYg0ichWEbnNGf4rEekQkXrn7wa/g3W1o0awVAaYnaRURBZ6pt/hXynf4BVKOZm0ted9Kj2AEQA/UUrVich7AWwUkXh3Zo8ppR7xLzzvTE540RGNhSwZNpZith+TdnCKtrwJXCnVCaDTed0vIs0ALvI7MNtwp6W4KNbNk5kKqgMXkXIAVwKocQb9UEQaROQ5ETkvy3cmiUitiNT29PQUFWxQFPdAl/Qtp6hUwxAFyXUCF5FzALwK4EdKqT4ATwP4awATESuhP5rpe0qpaUqpCqVURVlZmYaQyTRSRFVKMd8lci+a25mrBC4iZyCWvF9WSr0GAEqpLqXUqFJqDMAzAK72L0w9WLAubZmeyBQWXqMhHdy0QhEA0wE0K6WmJAyfkDDaVwFkf0xFyFjKIwD4YoZHbSXiAZ5s46YVyqcBfAfAFhGpd4b9HMDNIjIRsYrQdgDf8yXCAnEfLB26Eq5JpWGTYiHzuWmFshqZK5AWZBjmq0J2WO4H0Re/8GnautZ1QZbJnPKx4k5Mbsg8vc+J2weVKCsSeLHYRK108cBHUWZFAo9iXSf5y/RVbXp8ZAcrEnhcKSfgUv7tQWFhPbqiuv9YlcDd4F2UwbO9iirXvm33LyNfGLRRRC6Bx2XqoIh0i+4yju4vI69MTCmRTeBEYeFJIAWFCVwX7rVUwqav3oWB4dGwwyg5TOBFMPGUiuyQbduRpHHs2cDumd+ERxa3hB1GybEqgbOQa5pwV4ip24OhYfmub2A47BBKjhUJPOiCiKmJwVQ6OgsrZpnr6qyMLZjINlYkcIouE2oJTIiB/BXVVcwETkRkqZJK4DxDpiCwKoaCYkUCL3Z/4AMdyDTcJkkHKxJ4XCnXVZbwTyeiLKxK4ER+YsVHcVhzFLzIJfD4NsQSa3DC6sxK31yzby1B12cnnmVyGzaTSZ23RS+BO8u2lKtbgmPGQvZjXZtwF6QBIUSGjvVp4nULN0+l/6CIrBCRJhHZKiK3OcPPF5FqEdnh/H+e/+ESkalK8YAzODKKoyfDuwPVTQl8BMBPlFKXAfgEgMkichmASgDLlFIfArDMee8r1rFREou3h1JMdlH07WdqcMXdS0Kbf94ErpTqVErVOa/7ATQDuAjAjQBmOKPNAHCTX0HasLFbnEusZ8HmQRFVu/tIqPMvqA5cRMoBXAmgBsCFSqlO56MDAC7M8p1JIlIrIrU9PT1FhGqeIA8sPED4j2d4ZBvXCVxEzgHwKoAfKaX6Ej9TsUv1GTd/pdQ0pVSFUqqirKysqGC9MumqMQXLzbq34QyPKBNXCVxEzkAseb+slHrNGdwlIhOczycA6PYnRH24o5YOruvg8QwmeG5aoQiA6QCalVJTEj56E8AtzutbAMzTH144uB26pW9JRWnnd/NbeIAhHU53Mc6nAXwHwBYRqXeG/RzAgwDmiMitAHYD+IY/IYa3c3MncydX+9iFWzqzfkbpTGxrHAVRXap5E7hSajWy//4v6A0nt1JOqLb+9O+/XBd2CESRFbk7MYm80nWip+uiOUvjlA8TOJU8pkmylVUJ3E1deK7ST5QulPmtkE6cwuvMyv/5cpNxr1SqOE3KI1YkcC8bRuLpJ09F/WLGcvWj4ym/f5mN22S+xWxSYvODiQcoKxI4ERGlYwInokjY2XMs7BACxwTuGBgeDTsEClvU6wAi7guPvp31M7+rP/b3nvR3BlkwgTvurWoKOwSKiKKOAwbWs1J+n3pweSjzZQJ37D1c+BE08UJU1ApvUfs9uZjw9B0iLyKXwEsp8ZS6Ytb1yu0hd23s4pjB4wrlY0UC97KjcuMvHV7Wdfuh4/oD0YybMOVjRQKPK+WkXMq/nYgysyqBs3qEbMDtlIJiRQIPuvRZyG3kpS3c5aR77lzrZBsrEnhYSrnWopBkVsxt4ToOzl4mIVle62DDgaDUCilRrYKMXAIvse3SCFF+5qhf21PY+eSncxtCjsBeJm3tkUvgug0Mj6K8sgqzN+wJNY6wD0zvtB7MMDTsNOSfsEpsQc32Txv3BTQn/yilsKy5C6NjJqXUYJVEAi+mhNjTPwgAeGJZa9pnJ0vo9vtvPVujbVqDI6OoaugM5jS+dPftyFvS1IVbZ9Ri6sqdYYcSmpJI4HG6S1VDI2N6J2ipVzd2YMnWA67Hf3hRCybPrMOa1kO+xZS3Xj6qlaIlpNspXHUcCacfklRhXFdw81T650SkW0QaE4b9SkQ6RKTe+bvB3zDJZA8t2oZJL210PX7n0dgOd/TksF8heRJ2NVUay44xpi2+UuCmBP4CgOsyDH9MKTXR+VugNyx/GLeDGiyqrRQWZzhTCKMwHtX+Vzp6T2LW+uKuFy3f1qUpmujLm8CVUisBHA4gloIcOjaI1m53/f/qePpJRPe3kvO9lzaitbtf6zTX7jyE+xc0j7+P6sHPjW89sw6Vr20pahq/W55+vYkyK6YO/Ici0uBUsZyXbSQRmSQitSJS29PjrQOhTPvD5x55C1+ckr3/30J5emxbwnei3JQuak4M6b34fPMz6zBtZZvWadrq8LGhsEPIyMZH2LnhNYE/DeCvAUwE0Ang0WwjKqWmKaUqlFIVZWVlHmeXrn9gRNu0cjGlMMUzgHTFlHS5OPXjMg2epwSulOpSSo0qpcYAPAPgar1hJTMheZkQA2WWuG4Ghkdx+2sN6DXsAmkUbN1/FAAwNOpv6ytTCk02ON3Ll0RkglKq03n7VQCNucYn+9i6D83duA+vrN+bc5xsp9MKCqNjKtSSpMkXN9e1xS6F7ejKfu3J1u1GhzAOPHkTuIi8AuBzAC4QkX0A7gLwORGZiNj6agfwPR9jDFWuum1zd7XS5WUfSkzoH/7lQlx07nvwxuRP6wsq4zyJipc3gSulbs4weLoPsRgtqhdBSpFI9iqx4VGF9kMngg0oInQVQEu5FF+oyN6JWcyZqNsNyJYNzYY6RS8XJIP4WWHGZWqhwZhmkprjcDs5Y34/IpjA4wtXR11isdNYsvUA3tjUUXQcXplanZq4/eup89X/Q4tKnkXs34nJwdT1l0uu5PbsqjY0dhzVPk9Xy8nCZemGp4uYpaSYg2389vKbrrxIUzTJNu/txY1PrvFl2jqczNPe2sYEZYqWA/249P3vDTuMgtxb1Zx/JMCOU0ZDRK4ErtOY4aWhP2/e79u0c+1DU6q3o6qhM/sIjocWbdMYkT4imUvYNuWNf/jtyrBDMEbHkZM4dGww7DBCwRJ4Biu3x+4YLWSHHhgew7DP7WNN8cSyHQCAC845K+d4R06YeVdeqnD6Qil+GlUNnfjy300ofkIFMu04t6KlBxX3LQ07jFCWi1Ul8GJvV3+0ejsq7s2/ojt6T3VP6XaO01fvwo2/N7c6I4q8lphNvThYqMkz68IOIUmYS9WmsyedrErgOhz0eKqVaeNMHdbU2edp2l6ZWK0ThkKXg8nLzeQbedzQkUdLNBd7YlUCj0rJyZtS/u36+ZknTerYrKtvILB5mfOrS4dVCdxPiftz4ulYrmZRYZeWwp5/sUr1tDdIH79/mfZpdvYOYNOeI9qnS4VjAnfJ9mRZKD9LkTYuyubOPsyNwIOAdVjffhhffeod36bvx4Hdwk3OlZJohRJGwnhpbbvmKaZv1d0Bnh4HIahHrGXbHnIljusfX6Vl3gPDo3hp7e6SfpJ6VA2E8JDzkkjgxci1m+WqXvnlvK36g0nRuD/Yi6a6bN4buxvvz5v3Y+J/PRcAMDAyhivuXqJ9XqalyadWtOIJl0+ciWqpMZe3WrpdH9xMW7cHQihQlUQViptTsmyjxL9rw85UXlmFvgFz+sHOtszizTTX7Dw4PizfXZs6SYiXw/sCehCJjVa0dONfnt8QeGsum0UugWtpxpQp4xuYwTM9E3SPRT3p5VqkNW2H/JtvAevStFJelPX02XE3pUnbROQSeFwQ+bbULmym8vPn92hsr+/XhL721BpcfHvV+PtvTF2Lg8cGS6Z1TXllVdowW377suYuY7t6KIRVCTyo9rUqxzsdjhwf8qVXNiDzDlSzq/DSrA07YuIds2Go29ObtJzW7zqMmTV7ippmaRcJgnPrjFo8/dbOsMMomhUXMYOusfSzBuXHc+rxZv1+jATYCuFfnt+A9ge/nHe8kQL7cgkzyTft70NPv/dSeqazBz6QIFwm3QBlC6tK4P/09Fp8/Wn/2p/G+bkZvVbX4WvyPjY4giVbD3j67qodB/OPpJnXA+PuQ8eLnsb49wMoHyilsKKl2/8Z5TCWZ7sz6UEFugVR3RnG4rMigcePzP2DI6jdHewdYLZt0z99dTMmvbQx4wXOfExqm5zvrGtVq/eDTRiXLv60cR/+9fkNmFOb+4HLxchXLbe+/XDOz7/9bE3RMZhSii6vrBq/oL+osROrd/SEHJE/8iZwEXlORLpFpDFh2PkiUi0iO5z/z/M3zPCFf8HS3fz3Ho7VC3tplmfGrufOm/XF9IXu37rMdsDf79TXn/CwXlq7+/GhOxZg7+HcLYzyXfjNVwJ/Z6d/LX/CsM659vMff6zDM6t25R2/o/ekp4JPmNyUwF8AcF3KsEoAy5RSHwKwzHlvBB0l5nw7im7/90+bcfi43r6zvZSECj2F1tnm/GTAd7HZ0DFavNAwe8NeDI8qLGzM/xANm5lwtvvFKW9rn+YDC5sxr96fRyvmTeBKqZUAUs+9bgQww3k9A8BNmuMqWjEF5hfeadcWhxtzN+7DlOqWQOeZSaH7z9BI7oue+c5aijmrSX52pODw8SFMnlmHY4Peb5Rp1nADSWvPsbREVJun6iKbQi8qk5mmvt2G22bV+zJtr3XgFyql4sWBAwAuzDaiiEwSkVoRqe3pMaMeKlPiyZZKtuSoV9RZhtNdInRbmjlyfGi8D4egS0C5e3rM892U90+uaEVVQydmb8hfx1zoceOldbtdj5vpMXdemzv+4GW9D2wwoICrlQkl9rAVfRFTxfbCrItSKTVNKVWhlKooKysrdnZa7Dl8PP9Ijj6ng6WwT7j9avN85T3VCRev/N8j2g+6X/a5pO68xVyAzXfw/OUbjTk/90uhF+zD3kYHhkeLSqrMx4XzmsC7RGQCADj/h9s+KgTLt3XhkOZ661z+9yubfJv2RidRJO5801fvyninXbG+89yplg59AyO46cn8j6HLVJWQWscfr/Zy81xSGf8nODbUuRdrfkMnBvNUq+Vif4k6+B/gNYG/CeAW5/UtAObpCcc8iaukN+Ehvf/2Qm3wwRTAy6aU+J0ZPl0HGBhO3sG7s9yMk5juHq3e7nr6+47kP1MxqLVkmow3GBkcr8lMOGT63SDCTTPCVwCsBXCpiOwTkVsBPAjgWhHZAeCLzvvArN+V+aJQdVNXUsnswFF93TuKABN/Xa1teqmCbj97/4JmdPcnL5+kJxEFGk26xES2eW+v5+lkattcUJvgLAsi8UYiv8VvsMpXis93UdjrgeCe+U342dwGb18OkV/bcCHL8b8/vMKnKGLy3kqvlLo5y0df0BxLVqkb7v0LmjOO990Xa3HhX5wFAHh14z48smQ7XvvBp6x4AszIqEJzZx8+MuEvtEwvX5PAaSvb0tq86jyIbO/qx+ub0ptOeUkimdonJ06n0NU76iKGfNvMvVWZt0Hd2g8eR0tXvy/TnvjrJdhwxxfzjjd9df421BQOK+7E9KLGKaW3dhXXMD+o09dZG/bi+sdX+Vayq2pIb0OcWl9cTFJM9aXHVhY5BQKSrxlkcujYII54vBbTe2IY/T73T57v5qFEus9Cgy637Sgy13gR2QSuS3yTCuoilK4belJ3hckz6zI2cbNV4u97ra7wZ1Umrk0vZ2i5DuyZEpHXs8DUawapPnbvUlx5j39Ve8WauT5774x1zoORF2zpxOd+s6KgZO9GXcAPXv6+5mafbliRwIs6Mguwx8WFhBUtPZi2MvzuJf0s8P9nnpYsifPOdnExKHlvOU8IdsZa9+20ATMubrml8www44FF3+Qz6jya/aLy1556B2NjCj+b24D2QydwXPNTmV5Z71+/M6awIoGnqi/wotY8l/1m3L/Ang7e890F6WbHT+190KTe6H48Z3POz4s5qKeWht/wcJtzrj7W/VqMxV7LydhNctDNKVPm15VwId2gzc8aVvQH7oWujSHopOZ2di/XFFbqzOejv1ocftMTF44cH8J5Z5+pdWef+nZb2rAHnIP5WJYZFVJ3vO/ISUwpoClkkIxon+6EYEpPhjaxsgTuxuoiuhvNxLSWLHf/uSnPGIXtDP0DI+gvoh+RoOiq783X5O4Vp+52VMORotAzxmT+JrWK+/ytP089QKQudSMOIBazPoG39fh75Tf407qol0L0/L6gllJQ618phZNDo0UVFLJ9tbHjKD71wLKMF8iH3bSp1Cj1wJn4dsnWrkBjiQLrE/g1jyZ3/9iV8mTr+RmazxUi6NO6xIQxMjqm/cq8bS79xcKMw02qr9dh9oa9+Midi9B+UP+de1/53WrsPzqAH832p0e8XPIdkJ5bs2v8wFPcmYq/Vu84mPUGwjBZn8DzWbk9/a673y9vdf39eJ4I44EOf3PHQtz8zDpP341KfsvWt0YxP6+QNVlslwRu57ekKVb63JlyRpnUNt+0ejwPUn9BpusPJvpf02vS7ur1o6+gQlmRwHXXk71R1NNcCldIv86pO3+NgUf9oPhVyhZhzWtYMh2DonBgCktkW6HooiOF7NLUhWoholAAX9To7eHMWmlYkPkedaZj9q3d/fjn59Z7nk+iNzfvxwlNF7SZmv3FBO5SMRvitQXcVv5Pf1ibNszrrdK2y9UHejGF89WtB3Hd5e93Ne6QhqfibNrjf93usy6e+eiWn10Xx859olC8MAMTeB4m1CV7aTpnQtymqm7qMrZkGPXVpuOAGDc4EuxzVE1kRR24X9w8lFf3DTOlzpQDS/yioSniB5Suo6ld/LpbYLqWq/Z+4F3Ub3tNxI8sNvPmqCBZUQL3ozvNn8zZjKsvPi9teGp9dfwBAcU8LDcMJjazC+NagC3ieS7XY9SCOGu4682tAcwlWb4OuwBgQ/vhtB41dZbmbWVFAvfDq3X70HYw/Sagzz/yVsbx3XSIRbl9/pG3cME5Z7oe39rWCQYePG2X6dqQG119+h7qYqKSrkIJ4uJSKXHTLvbgsdK8IOtV4qHAxuNZ2CF7vY/CFiWdwKMsCmXAbDv/mxHq1zyf3hOnrtPk7/+GUnW4eEZqqnkeeqcMCxM4GSvbE+b9bOYWnuLKqrNrzez72sazhttmBd/lgFdM4BEVhWrYBxba0z97oqbOvrBDIIeNB5BCFHURU0TaAfQDGAUwopSq0BEUFW9d2yFXV/dJPy9Pgol6otGttl1PFxOjYwojFncYp6MVyueVUno736aiPb5sR9ghUAGqDWuXrotfvc583WOrlFRffmIVth3Q30wZCGadsgqFiCIr3wHEr+QNAN99sda3accVm8AVgCUislFEJmUaQUQmiUitiNT29KR37UpERN4Um8A/o5S6CsD1ACaLyGdTR1BKTVNKVSilKsrKyoqcHRHZ5LGl4d7ufnI42v2lFJXAlVIdzv/dAF4HcLWOoIiIKD/PCVxEzhaR98ZfA/gSgEZdgRERmWDbAXObhRbTCuVCAK87/VWcDmCmUmqRlqiIiEK0qPFUx1n9A+kd2S1rNqPVkOcErpRqA3CFxliIiIzQ3HmqdUqmm+I27zsaYDTZsRkhERktjK6RE+f4jal62pz7gQmciIwWSl/8lvRFwQROREYLI5Xmu7velAemMIETkdGeXdkW+Dx39qQ/7CXR75a3BhRJbkzgRGS0J0JIlgsbDwQ+Ty+YwImIAuD14c25MIETEQXAy9OB8mECJyIKgB8P6WYCJyIKgB89ozOBExEF4F0sgRMR2cmPx+YxgRMRWYoJnIgoAO96F6tQiIis5EP+ZgInIgpCvgcse8EETkQUAJbAiYhsxQRORGQnVqEQEVnKuCoUEblORFpEpFVEKnUFRUQUNYbJ3IQAAAYTSURBVEb1hSIipwF4EsD1AC4DcLOIXKYrMCKiKDlyYkj7NIspgV8NoFUp1aaUGgIwC8CNesIiIoqWf56+Xvs0i0ngFwHYm/B+nzMsiYhMEpFaEant6enxNKPrLn+/twiJiAwx9Tsf0z7N07VPMYVSahqAaQBQUVHh6Umgf/DhhxMR2a6YEngHgA8mvP+AM4yIiAJQTALfAOBDInKxiJwJ4JsA3tQTFhER5eO5CkUpNSIiPwSwGMBpAJ5TSm3VFhkREeVUVB24UmoBgAWaYiEiogLwTkwiIksxgRMRWYoJnIjIUkzgRESWEqU83VvjbWYiPQB2e/z6BQAOagzHL4xTP1tiZZx6Mc5T/kopVZY6MNAEXgwRqVVKVYQdRz6MUz9bYmWcejHO/FiFQkRkKSZwIiJL2ZTAp4UdgEuMUz9bYmWcejHOPKypAyciomQ2lcCJiCgBEzgRkaWsSOBhPDxZRJ4TkW4RaUwYdr6IVIvIDuf/85zhIiJPOPE1iMhVCd+5xRl/h4jckjD8YyKyxfnOE+Lhiaci8kERWSEiTSKyVURuMzFOZzrvFpH1IrLZifVuZ/jFIlLjTH+20zUxROQs532r83l5wrRud4a3iMg/JAzXtp2IyGkisklE5psap4i0O+umXkRqnWEmrvtzRWSuiGwTkWYR+aRpcYrIpc5yjP/1iciPTIszjVLK6D/EuqrdCeASAGcC2AzgsgDm+1kAVwFoTBj2MIBK53UlgIec1zcAWAhAAHwCQI0z/HwAbc7/5zmvz3M+W++MK853r/cQ4wQAVzmv3wtgO2IPmDYqTmc6AuAc5/UZAGqc6c4B8E1n+B8AfN95/QMAf3BefxPAbOf1Zc42cBaAi51t4zTd2wmAHwOYCWC+8964OAG0A7ggZZiJ634GgH93Xp8J4FwT40yI9zQABwD8lclxKqWsSOCfBLA44f3tAG4PaN7lSE7gLQAmOK8nAGhxXk8FcHPqeABuBjA1YfhUZ9gEANsShieNV0S88wBca0Gc/wVAHYCPI3YH2+mp6xqxfuY/6bw+3RlPUtd/fDyd2wliT5daBuAaAPOd+ZoYZzvSE7hR6x7A+wDsgtNgwtQ4U2L7EoA1pseplLKiCsXVw5MDcqFSqtN5fQDAhc7rbDHmGr4vw3DPnFP3KxEr2RoZp1MtUQ+gG0A1YiXRXqXUSIbpj8fkfH4UwF96+A1e/BbATwGMOe//0tA4FYAlIrJRRCY5w0xb9xcD6AHwvFMl9ayInG1gnIm+CeAV57XJcVqRwI2kYodRI9pgisg5AF4F8COlVF/iZybFqZQaVUpNRKyEezWAD4ccUhoR+QqAbqXUxrBjceEzSqmrAFwPYLKIfDbxQ0PW/emIVUU+rZS6EsBxxKoixhkSJwDAubbxjwD+lPqZSXHG2ZDATXp4cpeITAAA5/9uZ3i2GHMN/0CG4QUTkTMQS94vK6VeMzXOREqpXgArEKtOOFdE4k+GSpz+eEzO5+8DcMjDbyjUpwH8o4i0A5iFWDXK4wbGCaVUh/N/N4DXETsomrbu9wHYp5Sqcd7PRSyhmxZn3PUA6pRSXc57U+OMKbYOxu8/xI7gbYidisUv+lwe0LzLkVwH/hskX9B42Hn9ZSRf0FjvDD8fsfq/85y/XQDOdz5LvaBxg4f4BMCLAH6bMtyoOJ3plAE413n9HgCrAHwFsZJO4sXBHzivJyP54uAc5/XlSL442IbYRSft2wmAz+HURUyj4gRwNoD3Jrx+B8B1hq77VQAudV7/yonRuDidac0C8K8m70tJ8RY7gSD+ELviux2xOtM7AprnKwA6AQwjVoq4FbG6zWUAdgBYmrBiBMCTTnxbAFQkTOffALQ6f4kbRgWARuc7v0fKRR6XMX4GsVO6BgD1zt8NpsXpTOfvAGxyYm0EcKcz/BJnw25FLEme5Qx/t/O+1fn8koRp3eHE04KEK/m6txMkJ3Cj4nTi2ez8bY1Px9B1PxFArbPu30AssZkY59mInT29L2GYcXEm/vFWeiIiS9lQB05ERBkwgRMRWYoJnIjIUkzgRESWYgInIrIUEzgRkaWYwImILPX/AabaEtMcrmLsAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(losses_ru)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "100 nodes max, 90 epochs I think"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x1f6702460>]"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXIAAAD4CAYAAADxeG0DAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de5gU5Z0v8O8vGt3nxGyiK8eHdU1GXc2zPpuNGtY1J15OYnRRk6ibZFfjiebEBHdX92iOnmRQk+MxmkUQUYM3QCIiGkVElBEQhvudGRiG28BcmAEG5goDM8Pc5z1/dPVY09NdXfeqt/v7eR4eeqqrq35d3f2rt956L6KUAhER6eszUQdARETeMJETEWmOiZyISHNM5EREmmMiJyLS3Klh7uzss89WBQUFYe6SiEh7paWlLUqpUZmeDzWRFxQUoKSkJMxdEhFpT0TqrJ5n1QoRkeaYyImINMdETkSkOSZyIiLNMZETEWmOiZyISHNM5EREmmMiJ8oRG2taUdXUEXUYFIFQOwQRUXBun7YRAFA74eaII6GwsURORKQ5JnIiIs0xkRMRaY6JnIhIc0zkRESaYyInorxX3dyBwUEVdRiuMZETUV7bWX8c101ehelraqIOxTUmciLKawePngQAbDvQFnEk7jGRExFpjomciEhzTOREFLiOnn58tP1w1GHkLI61QkSB+/W8chSVH8GFo87AJX/551GHk3NYIieiwB1p6wIAdPX1RxxJbmIiJyICoMB25EREWhKJOgLvmMiJiDSnbSIvrTuGqcsrow6DKO+sq2rB4p1Hog6DTLRttfKDl9cDAO7/9kURR0KUX+6csQkAZyKKE21L5PnqWGcv+gcGow4j762vbsGcTXVRh0EEgIlcK919A7jsd0vx2w93RR1K3vvx9E14dP7OqMMgAmAjkYvIeSKyQkR2i8guEXnAWP64iNSLSJnx76bgw81vXb0DAICPd7B+kshvSt/Wh7bqyPsBPKSU2ioinwdQKiJLjeemKKWeCS48IqKg6d/+MGsiV0odAXDEeNwuInsAnBt0YEREZI+jOnIRKQBwGYBNxqL7RaRcRGaKyJkZXjNOREpEpKS5udlTsERENJLtRC4iZwCYB+BBpdQJAC8DuBDApUiU2Cene51SappSaoxSasyoUaN8CJlopA+21eP88UXo7huIOhSi0NlK5CLyWSSS+Byl1PsAoJRqVEoNKKUGAUwHcEVwYUavsrGdSSLGJi6ugFJAa2dv1KEQhc5OqxUB8BqAPUqpZ03LR5tWuw1AzrbF6ujpx/VTVuOX75RFHQoRBUTjRiu2Wq18E8BPAOwQkWQmewTAHSJyKRLvvxbAvYFEGAPJkvjm/UcjjoSI/JYLg2bZabWyFunb53zsfzjk1Yw1NRhTcBYuPe+LUYdiW2//IKqaOjjhQB7Qua12nOVFz06lFD7YVj/UoSaXPVm0B7e+uC7qMBx5YuEu3PTCmqHZzCn3SC4Ue2MsLxL55v1H8eA7ZXhi4e6oQ6E0th1oAwAc7+qLOBIiPeVFIu/oSUwv1XiiO+JIiIj8lxeJnIgolzGRU05RvJtGLun81WEip1h7fd1+FBRm77HJm2nkVi58c5jIKdZeWlkNgDdCiaxon8h5A5OI8p32ibzhOBM5BaO6uQPPLt3HeneKPe0TeRjs/o7nlR7CzvrjwQZDofnJjE14obgSLR0ciCuXqTSPdGNnrBUyZLuf9tDc7QA4u3iu6M2xSa57+wdxymcEp3wmF27v+Sc5deKyPU0RR+IeS+REeeLixxbh57O2RB1G7Bw61hV1CJ7ldSLvGxjEw3O3Zx3jQ2l8yUVktmIvZ+lKlQvXJ3mXyFs6elBadwwAsLGmFe+VHsL493fYfHUufOS5jfclKR/lXSK/Zeo6/ODl9VGHkTcKCovw2Ad2T5S5r+1k71CdLJFf8iqRK6VQ36Z/fZhu3tx4IOoQYuO+t7bi3+ds5feQfJUTifyTXQ3o6c/chZu9t+Mtn6pDkjfW+vpzq0UMRUv7RL6xphXjZpdi0uK9WdfNo3xBFEv8DQZD+0R+7GRiDA6rJkQS4E3K1o4e7DocTCeg4yf70HaSnVFIf7woDpb2idxv3568EtdMXGF7/RufX4ObX1gbSCxfe+ITXPrE0kC2rRtOA0eUWc4n8paOHuw+csL2+jXNnTjgIGk0tfe4CStUK/c2oaCwCE0aDjCWvBT/4SsbIo2DKM607KLfn6XrdGtHDxSAs884Hf84ZTVaO9NXT9i+yaZ5xd7sDXUAgPJDx/GdS/4s4mj0ww5hFHdaJvK/fnSR5fNff3IZgMSYJ5mSuJlVq5YluxrwYdnhrOulU9FwAg3Hu/Hfv/Jfnb2QYoI1u6QHLapWapo7UFBYhKqmjtD3fe/sUhQ57MCRnARh7HNr8NM/cmyLMLCJKbmVC9dbWiTyBUaJ+KPth7OuOzgY/cdy35ytUYcQmicX7sa1k+zfHCYi/2mRyJ3YdvBYYNtubu/BhurWrOvVtnYGsv/oT1EjzVi7H3WtbFESV3WtnVhf3RJ1GNooO9iGbz2zEh09/VGH4kjWRC4i54nIChHZLSK7ROQBY/lZIrJURCqN/88MPtzsgu4l+MCftgW7A8ppYfdivXbSSvx4+qZwd6oZc63c04sqsL+lE+UH2yKLxw07JfJ+AA8ppS4BcCWA+0TkEgCFAIqVUhcBKDb+Dl0+tShgNbD+kt9X1umTn7ImcqXUEaXUVuNxO4A9AM4FcAuAWcZqswDcGlSQlqnaeHJH/XFc/+wqdPZmHnMln8b00Ek+fixB9jYmZ3Lh++eo+aGIFAC4DMAmAOcopZLNORoAnJPhNeMAjAOAL33pS27jNLaV+bnkaHJhXhL9fFZJaPvKByylErlj+2aniJwBYB6AB5VSw7pKqsQ042lPbEqpaUqpMUqpMaNGjfIUrFMr9lrPwef1TLxsT6PHLRAReWcrkYvIZ5FI4nOUUu8bixtFZLTx/GgAkcxcapWMX1uz3/f9DbJ+hohixk6rFQHwGoA9SqlnTU99COBu4/HdABb4H152yV6XfntpZRUKCovSPMPrf7cWlh/G9NU1I5YrH0+OPM+SU+ZftK6NJ+zUkX8TwE8A7BCRMmPZIwAmAHhXRO4BUAfgn4MJ0Vpzh/tBq5IfYG//IAoKi/DQ9RcPPTcxw/jmyXrck716tTONg/vfSjTd/MU1F6R93ssNwEDr1/X8bZMXmpXXsiZypdRaZH5b1/kbTsYgfNnMqn3DZxBPbvWk0dJlxlr7VTE3TFntS0wUX7z56r+4XzHFPb5MtOrZ6XeTreSogDvqnU8MYTWRBfknyh+Wrj/qONLtpKhb81CtEnk6Xg53UF3p42alcSXS43GeyK7eAQzEYCyb0On1myaHcuEbrVUin7Js34hlcfwQ4lZaTybfQ8e8jYnyN79djPHvl/sRUt7SrZTf2dOPgsIiTFtdHXUogTlp0YlQF1olcr+5aS1ht3CWbfILvzz07nasqxo5KNL9bwUzAuO7JYcC2W6+0aWq4agxnv+s9XURRxKM2pZO7HEwg1g6czbVoSzisVm0SORW6Tb19zB56chSexSunbQylP3M23oId84YOSjSwnJnY6jno8FBhYbj+k1/R/6pbh4+x4GbC6ZH5+/ErS+u8ycgl7RI5GGxU0pq7ujBpprsQ9kmhwyg8HX3DWDm2v1Z6/NfXlWNK/+zGLUt4d0r0a1qJV/pcsWUxERuYuezUwr4l2kbPe9r7HOr8ZsPdnreDg2noPB8cSWeWLgbC8rqLdddbdwEPsJSeSS+8Z/FmLi4IuowcoL2idzqzJntrCoRnnYrGtoxe2Pu1TtuO3AM7d19oe/X3FwsOdVeLtzEymVHjnfjpZUxu4mq6RWT9oncSrbL2MYQS2JdvQPYemDk7EWvrKoO9dI+SF29A7jtpfW4d3Zp1KGQT/wo6wwOKmypDW7mriBoVrOiRyI3J+R9je2+bHNTTSvaQ5zO6eG52/FPL61HU/vwk8eERRX48XTvVTV2BH0B0j+YaKlTfsh5ByuzxhPdaDvZ60dI5BMv4+EEOf2iV7lyz0KLRG6W2jXebQ+sioaRJ4Qgq1qSvUe70lzud/WFUwUQ995qv3ijBN19A/iH3xfj8t8tjTocHO1MjONT1fRpy4b6ti7sdNETWFd+/Cbi3IfsyPHcaJSgXSLXkd3SZUtHD+6dXRJaHfPAoEJBYRGejUmTzfq2LmypPQognB9/tl0kY5hhGg75mxOW47t/WBtcUBQpXUc/zOlEHpfu5Mv2WA/VfuxkH1bta8bU5VVYsqsRc0PqdNNrdNnP5V57dujW1Myud7Yc8G1b8fglhcfulcgOj9WIftEikVc0WPS8sjjeG2y097a5qcAt3B7MuOpByNYTzs/xxa2EWY8+6HOhIIyTx6/n7fC8Db/DfHVVNa6bvNLnrWZX19qJ9dUje0B7/VQtc1OItEjkViXaXpcDQbV6GMc8nWxnZl0v2dLJ1BsyWYrx952m39qiHUdw6RNLUVp3dOQr7AZgc70d9cdxwSMfY01lc/aVyVJxRROqm8NvpXXtpJX48fSRPaB5s1NzLyyvGrHMSynpe1Ot600PHk3cVIn7DUcv/Hhndn9YyautTfuPorm9Z6g1kPn1dj/PbKs1tydO+ssrIpnNMBAdIbbY0omuiT1vE3mUntNkFMc4Sw7mNHHxXvz9U8vQ3RfOIGXZPDJ/B66euDzqMCwt3d2Iv/2/S1BaZ79ZoK4JLpu+DIPb6XbfhIk8ZD39A3huWeWI5WH8UFK/nEFV9wT1XsytefwYFCyI9//WpgNDV19xlRwts/xQ9hH7kt+ZXKoaNJv8yfBCla7vkok8BOabf9fHcIo4p9U9mX7UQZdinJQgnQhzqIawbgT7JYyqwChbl2Xqw6FZgZyJ3KylI7rehHWtnXh9fW1k+3ciyFzkddODSg3VafvNz/cd5Tg/cfPrecFMVjI4qDxPppJNXE7LTOQxURJQaTMIv/94D+6c4WxYAXMroe6+AczeWBdI6fSumZuxbE+j79uNm4LCIjz2gffmhXHwXmkw/SZeXlWNq55eMWLM8VzERJ7D/rhuP4oDSGrVzZ1YV5W5jX66qpe6o5+WjJ5Zshe/+WAnFu9sSKxvWn3QIrlPX1ODxhPWA53FZZq9u2ZuDnx0yzc3uuvw4+YE2niiBw/P3T5sWXdIQ0u4taE68R09nAdzAzCR57D/99Fu3DOrxPHr6tu6UOOiFJOsT+3uG7QcZuDYycRz6ZrATU3TLDRpXVUr/vVN/0ZWDLKKaPW+Zk/jzR88ejLwumOnlTupJefWTg5sFpe6FSbyELSdDGbsFLcloqb2bmw7cGwokaXe8PnmhOX49uRVjrdrrvb90SsbXMWW7YZmp4P2z9lu1CV/g8mEuaX2aNYSv1d2fve1LZ24euIKvFA8snWTnzHsPnICzyzZa7kuq/L1wEQegqc+3hPIdm+Z6myewOQNtpueX4vbXlqfNakMDCocaLV3syhZTZKUOrpkHBtrJEcxnGIMGvajVzaMGF0zKFb5MTlj0UaHQ0w49W7JIUxdUWU5UbjOedxNk8lktZNuJ7CsiVxEZopIk4jsNC17XETqRaTM+HdTsGHml0xfv9Tlex2OzZ78brbYHJ5gytJ9uGbSCtS1Zu9S/cwn1iW7dKLO7T3G8A57TONlJGcX8ssv3ykbardNmf2fuduHOnn5zUkTyk+/k3plcjsl8tcBjE2zfIpS6lLj38f+hkVxkCwRNvncnE+30o4X87fV484ZI8f4yCd2rsbmlh7CZFNh4J7XtwTWmsVPcekolTWRK6VWAxg5MhEF5ncLd2Nt5chSXK7lv1+9Vx6boYbziV/VXH5/H80n+OKKphGtZOKoPybfXy915PeLSLlR9XKmbxERAHdVFU7FoZfhyV5ngzfl8qBjSWGX8hSA/oFBvL35wMgTq4fJzaMWxtf70fnuWyb5yW0ifxnAhQAuBXAEwORMK4rIOBEpEZGS5mYOA+pFTUvwHRvMc4qWm6Y0szNcsJvRBict2WtZLx2Dc01gNtW04u8eX5Lx/ac7njdMWYV3Sw76HsusDXUY//4OvOmg7Xtzew8KCovw1ib/JrAIQtxPOH5wlciVUo1KqQGl1CCA6QCusFh3mlJqjFJqzKhRo9zGmddeWVWNj7Yfxg9eztykr7d/0JfJD654qnjYNgGgp28Q//H2VsvXuS3dv7GhDg0Omvw5+VHabZ5pd5N+X8E8X1yJE939juYA3dfYgV+9561Le7oSf3KSDic3e2uNFk2PzN+Be17f4ikm8uZUNy8SkdFKqeTwc7cBiMf1RY6asKhixLLUpHLxY4tw1ze+bLkdtyWTf3uzFO0ajl/9xMLduP2K8/BfTrP+moc57smRDJNyuDVjTQ3+vuAsfO28L/q63SS7VVnFEYzVXtFwAuurWvGzq863XM/JzWZdrwDtND98G8AGAF8RkUMicg+AiSKyQ0TKAXwLwC8DjjPvlB3MPMRo28k+nD9+ZEOhOQFd4rpN4ieG9e6M5hfS0f1p7Mc6e1FQWIQFZfWRxGLm17njyaI9uOXF4f0JslWDpSYr8991NvsNANHffB/73Bo8sXB3xufdJOXkS3SrjrHTauUOpdRopdRnlVJ/pZR6TSn1E6XUV5VSf6eU+r6pdE4RytYCJOjvZmrp7R9M1TTD1ovoR5K8xzBrfS1qW8Kfbsw2j+e8dWnmprTe3ac7nLd1eJM/Pz4ru1VScbqRHZ9I7GHPzjyS+nPyWkYuKCyyfD7TWM9xcDDg4U2dUCox5OoH2+rRPzD4aalQu3TizdYD+owACsRrMC5XdeSUG/ysD1Twv6T9XumhWA3M5GflkECGHa/3t9Xj4bnb0XCiG1899wuJdWKex/2Ob9fheMxIb9d/mxCfKf1YIqfYStchxOmNSXPyTZ64mlOGKPC7W74bx4wTVktKL9rDbV2Yvy34Ho5uTupxv2KIS6/LMLBETqGLOgHEfU5Nc1L951c34NCxLtz81b/Eaaf6U+4aUcVmke/inaopiSVy8o3dH326cciDVFp3DAWFRdrNFCOCQIbVTZe4M13o5N2UdMbBue2l9Vi8swEFhUVo8LnJaBCYyPNIXH6SRTvCa+TU2z+Ie2cnJqNYvc9+a47jAY0hn46dKoCg556kkf7X29sAAJv2BzucsB+YyPNZiFWIUQ3UdPXEFbaH7TX72hOfjFjmd2cRJ1VM3568CpscjE/e6uI9O5WpsH7d5JW44qllkU8Fp2vnHjeYyPPI4x9l7jzhmbK+DM+j31Rg9jXZrxr6+pPLPO2rouGEZac0IPNJtbq5E03tPXjWmLBDJ9m+p8V7GlEfo2aHSUzk5Bur8uXzy4KZtsytOJfWzFUtYcSZbh9jn1uDW190NgNVqtaO+DQddSNdldc9s0rw3RfWRBCNNSZyCsVan2bJ8XLvLajmaG9sqHU0amCqTMlaIAFFPHKrmWPIX5mOybEQ75/YxeaHlJf8bIzx2wW7AAD/40rrQctS9x9VgxAnpXxfuuh730Ro+zUfmxhftI3AEjnlDa9zQobVwUSX0nG+tEzUoQkmS+R5xjw+ip+JKazSi5fflN9DyALZx5vJxM7bSH2vQR1jBRWbpKyUwoGj8WhqGYcZtOxiiZwAAFOW7tOi44Nf4vwbTYa2prIl5nOa+p/9X1u7H9dOWun7dnMdS+R5zJzMni+uxIZqbx0f4lKqC8qxzmBuctlN1V4Or7ktfbaTmPnkEfZwCqV1mUdAXLq7EVVNHbjnqvPtDVfg4hzYbhpDP86n0FRM5DSkpz++w84GraIh+8h7YfZIdaqysd3y+dkbMreqUWp4cn+v1P85Qf3wizdKAABtXb0Yf+PfBLKPWtPEGsljokP5hFUrpBW/SoipVw9jnwu+bbCTnpnpWJUQr5+y2sF2rMuaJ3ttznUaUoZTSuGNDbVDfx865r5DTkFhEZocjl+jw5UmEzn5QqcbQwDQN5B5OrRPdjX4vr/Wjh78y7SNvm/XjW0HhvfYtPzkLJJY6tAHqZ2+/LqZXnawbaiJpx+2HrDusZrJ/hjPKsVEnsdG/Mw0KHr4FeIflldlfG6cMciWn773h7WWz/9+0Z6hxxlPih5OluZRFCsddPW38uj84XOuT1nmvUv+4jQn0Z4sc5Bm4ndz0R+9st7X7fmJiZzyUmPILXQOp+zv9fW1Q4+VUoG3oikqt67fzzyMbQDBWIjjhV2yOi/s4ZedYCKnIdlmX9ddnJLESyurhzqa/PSPW2y9xlP4DhJy/K/L4qm7bwDj3y/33PHMDSZyGrLniLc5E3XoAZekU6x+sHq3Vvc3cv3kbofdr8qCsnq8vfkgnl5UEWxAaTCRky/stnSIizjfnA1iViArL62szni1Mmnx3mF/R9FBye0pN+yPOMqvFBN5HvMzmTW1Bz+RAeAtkcSl63dSf4aWM7+etyPt8qASxcsrq4f93W0qhTe1Dz+p3BzDIVzDEvVcs1aYyEkruw6fwAN/2uZ5O3GoWlnvsSdtUCaYqgZSj1NFg3XHo3Tid/HjLKDkEbD7lVFQWLm3CYMhXr0wkZN2FpQdjjqESHi5gorDicsv2d7J8a4+FBQWocSiu3+QluxqxE//uAWvrd0f2j6zJnIRmSkiTSKy07TsLBFZKiKVxv9nBhsmBSF2BSUKTA7l8ayyD/7m7GA4PXbHuxLjtRwMccJsOyXy1wGMTVlWCKBYKXURgGLjb9JMZaM/HUMoXrZnmWsznUASvV8TbufTWcilrIlcKbUawNGUxbcAmGU8ngXgVp/johDcMT0eXcbJX7U+diX3kkK3HkhUbYQ1IUfQ4nw6cVtHfo5SKtlVrAHAOZlWFJFxIlIiIiXNzc0ud0fkr87e+PbSyyTIdBjEDUnzSIJ+iqoBpNMrgzBv8nq+2akSd2AyhqyUmqaUGqOUGjNq1CivuyPyRVsMJ9DNxm5iSNeVPM6lSSsltamVASPFe/KNcLhN5I0iMhoAjP+b/AuJiLxYtmfkzzFbaXLqivSDiEVdPf3DVzZkXaeztx/9A4O48JGP8fbmA77HkO4YxG3cFbeJ/EMAdxuP7wawwJ9wiChO/Bop0YtsHbnKDrShq28AA4MKTxXtsVzX7Nml7kdrnJmmaWHh++k7coXBTvPDtwFsAPAVETkkIvcAmADgehGpBPAd42/Kc8mbW/nKj56ycaskqAuontvJtIJrKoffWzvR1YctpiqXu2ZuHvZ8XWu2m72Jo/xCcWWW9TIbtPFZh3mTN+tUb0qpOzI8dZ3PsZDm5mz0/7JWJ6srWwLdftySvBdeWkytqWzBGotjvaP+uOttpyMp/8cRe3YS+aSjO9h6U089O32MI46S9zs7evqDe68xPohM5EQ++c2CndlX8qDVwzjXUd+09MJO6Fv2Z2/d4myLemEiJ/KJHxMKWI3PkTpKYb6w0357IMBG2yKClo4etJuuuOI2EFjWOnIiIreW7GrA5E/2Zl8xxgTAmCeXOX5dmMmeiZx8MzDI2WRouHsDmMg6nWFl9hDqkeJWVcWqFfLNB3k6vKweYpZ5fJbvA2sxkRNRrNlJ0cPWyVqn4azO4+dvlDhaPwpM5ER5QOsCq43Yw35/duq/w7wfykRORLklhKwes0YrTORElFt0vvhwi4mcKA9ondxsFH+dFMJXVDTHbvRCr9j8kIi0Jw5OVe+UHByaVzNIWk0sQUTxl+s3O52qzTpCol6YyIko1myVth0m+94Bj53XYtZHn4mciLTntNAe9EiVYWMiJ8oDTuqQKbt4lceZyIko5uzU7zue4d5lLM6El+6ZyInygM43Ox130c9DTOREeaAzx9pNWwnjpGU1yUcUJ00mcqI8cCLHbu5ZCeN+wKYa+5NHh4GJnIhizU4J966Zmx1ts7m9x2U02SXDZYcgIiJDHFvcWN1cjWJsdCZyIsopfV47+3g0MJgoim/efxQXP7YIC8sPY8muhkD3ybFWiCinTF1RFXUIAICalsQwAPe/tQ0AUDvh5sD2xRI5EZFDcavsYSInItKcp6oVEakF0A5gAEC/UmqMH0ERESXp3JkpLH7UkX9LKdXiw3aIiLTAsVaIiBxgiTw7r4lcAfhEREpFZFy6FURknIiUiEhJc3Ozx90RUZx19w1EHUIo4nZu8ZrIr1JKXQ7gRgD3icg1qSsopaYppcYopcaMGjXK4+6IKM6mLNsXwFbjljaByqaOqEMYxlMiV0rVG/83AZgP4Ao/giIiPbV1+j8XJqtWsnOdyEXkcyLy+eRjADcA2OlXYESkn+KKxqhDyEteWq2cA2C+Ma7AqQDeUkot9iUqItJSS0fm4V3dajje7fs2c43rRK6UqgHwNR9jISIaYXlFU9QhxB6bHxIRaY6JnIhIc0zkREQhOHTsZGDbZiInIgpBZWNwbc+ZyImIQhBke3gmciKiEAQ5BRwTORFRCD7DEjkRkd6CnESaiZyIKAQskRMR6Y6JnIhIb6xaISLSHJsfEhFRRkzkREQheHvzgcC2zURORBSCw21dgW2biZyIKATs2UlEpLkgpx5lIiciCsFnWCInItIbmx8SEWmOJXIiIs2xRE5ERBkxkRMRhYDND4mINMdhbImINHeg9WRg22YiJyIKQU1LZ2Db9pTIRWSsiOwVkSoRKfQrKCIiss91IheRUwC8COBGAJcAuENELvErMCIissdLifwKAFVKqRqlVC+APwG4xZ+wiIjILi+J/FwAB01/HzKWDSMi40SkRERKmpubXe1o4X9c5S5CIqKY2PzodYFt+9TAtmxQSk0DMA0AxowZo9xs42/P/QJqJ9zsa1xERLnCS4m8HsB5pr//ylhGREQh8pLItwC4SETOF5HTANwO4EN/wiIiIrtcV60opfpF5H4ASwCcAmCmUmqXb5EREZEtnurIlVIfA/jYp1iIiMgF9uwkItIcEzkRkeaYyImINMdETkSkOVHKVR8ddzsTaQZQ5/LlZwNo8TEcvzAuZ+IYVxxjAhiXU3GMy6+YvqyUGpXpyVATuRciUqKUGhN1HKkYlzNxjCuOMQGMy6k4xhVWTKxaISLSHBM5EZHmdErk06IOIAPG5Uwc44pjTADjciqOcYUSkzZ15ERElF007j4AAAWASURBVJ5OJXIiIkqDiZyISHdKqdj/AzAWwF4AVQAKA9j+eQBWANgNYBeAB4zljyMxxnqZ8e8m02vGG/HsBfCP2WIFcD6ATcbydwCcZjO2WgA7jP2XGMvOArAUQKXx/5nGcgHwgrGPcgCXm7Zzt7F+JYC7Tcu/bmy/ynit2IjpK6ZjUgbgBIAHozheAGYCaAKw07Qs8OOTaR8WMU0CUGHsdz6ALxrLCwB0mY7ZK273bfX+LOIK/DMDcLrxd5XxfIGNuN4xxVQLoCzM44XMOSHS71bG36TfSdHvf0gMkVsN4AIApwHYDuASn/cxOnngAXwewD4kJpR+HMDDada/xIjjdOPLW23EmTFWAO8CuN14/AqAf7MZWy2As1OWTUz+gAAUAnjaeHwTgEXGl+pKAJtMX4wa4/8zjcfJL+BmY10xXnuji8+nAcCXozheAK4BcDmGJ4HAj0+mfVjEdAOAU43HT5tiKjCvl/LeHO070/vLElfgnxmAf4eRcJGYt+CdbHGlPD8ZwG/DPF7InBMi/W5l/B26TX5h/QPwDQBLTH+PBzA+4H0uAHC9xZd8WAxIjMn+jUyxGh9UCz79IQ9bL0sstRiZyPcCGG36wu01Hr8K4I7U9QDcAeBV0/JXjWWjAVSYlg9bz2Z8NwBYZzyO5Hgh5ccdxvHJtI9MMaU8dxuAOVbrudl3pveX5VgF/pklX2s8PtVYT6ziMi0XJOYGviiK45UmJ0T+3Ur3T4c6cluTPPtFRAoAXIbEJSAA3C8i5SIyU0TOzBJTpuV/AaBNKdWfstwOBeATESkVkXHGsnOUUkeMxw0AznEZ17nG49TlTtwO4G3T31EfLyCc45NpH3b8DIkSWNL5IrJNRFaJyNWmWJ3u2+1vJejPbOg1xvPHjfXtuBpAo1Kq0rQs1OOVkhNi+d3SIZGHRkTOADAPwINKqRMAXgZwIYBLARxB4hIvbFcppS4HcCOA+0TkGvOTKnHaVhHEBWOKv+8DmGssisPxGiaM4+NkHyLyKIB+AHOMRUcAfEkpdRmA/w3gLRH58yD2nUHsPrMUd2B4QSHU45UmJ7jelht296FDIg9lkmcR+SwSH9gcpdT7AKCUalRKDSilBgFMB3BFlpgyLW8F8EUROTVleVZKqXrj/yYkbpJdAaBRREYbcY9G4kaRm7jqjcepy+26EcBWpVSjEWPkx8sQxvHJtI+MROSnAL4L4E7jBwqlVI9SqtV4XIpE/fPFLvft+LcS0mc29Brj+S8Y61sy1v0nJG58JuMN7XilywkuthXKd0uHRB74JM8iIgBeA7BHKfWsaflo02q3AdhpPP4QwO0icrqInA/gIiRuXKSN1fjRrgDwQ+P1dyNR55Ytrs+JyOeTj5Goj95p7P/uNNv6EMBdknAlgOPGJdoSADeIyJnGpfMNSNRfHgFwQkSuNI7BXXbiMhlWWor6eJmEcXwy7SMtERkL4FcAvq+UOmlaPkpETjEeX2AcmxqX+870/qziCuMzM8f7QwDLkyeyLL6DRD3yUBVEWMcrU05wsa3Av1sA4n+z0/i8b0LirnE1gEcD2P5VSFy+lMPUDAvAbCSaB5UbB3e06TWPGvHshamlR6ZYkbjLvxmJpkZzAZxuI64LkGgVsB2JJlCPGsv/AkAxEs2TlgE4S316Y+hFY987AIwxbetnxr6rAPxP0/IxSPx4qwFMhY3mh8brPodEqeoLpmWhHy8kTiRHAPQhUc94TxjHJ9M+LGKqQqKudFizOQA/MD7bMgBbAXzP7b6t3p9FXIF/ZgD+zPi7ynj+gmxxGctfB/CvKeuGcryQOSdE+t3K9I9d9ImINKdD1QoREVlgIici0hwTORGR5pjIiYg0x0RORKQ5JnIiIs0xkRMRae7/A9rllKCVld0fAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(losses_ru)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "400 nodes max,    90 epochs "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x1a45ee430>]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXIAAAD4CAYAAADxeG0DAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXRc1Z0n8O+PJaQDTGPAw/HQBAGHppvpM2FREyCBbEADWWjS3TM4gRBCx8MAp0mHbtrEYQmYxo3ZAmYzXjBgk4CNY4PB+77JlmxZli3LkoUkW7tk7XtJd/6oV3apVK/evpW+n3N8LL169e5Pr6p+79Z9dxGlFIiIKLpOCDoAIiJyhomciCjimMiJiCKOiZyIKOKYyImIIu4kPws7++yzVU5Ojp9FEhFFXkFBQbNSarze474m8pycHOTn5/tZJBFR5IlIVabH2bRCRBRxTORERBHHRE5EFHFM5EREEcdETkQUcUzkREQRx0RORBRxTORERC7oGxzCooIjCGJqcF8HBBERZatnPyvBvG1VGH/6Kbj+L3UHYXqCNXIiIhc0dvYDALr6Y76XzURORBRxTORERBHHRE5EFHFM5EREEcdETkQUcVmXyDv6BrG5rDnoMAAAszZVYHlxvSfH7h0YwgPzd6G+vc+T4xNRdGRdIn9g/i7cOTsPR7sHgg4FU5eV4L73CxwfJ90Ag2V767Bsbx2eW3HA8fGJKNoME7mInCci60Rkv4jsE5GHtO1PikiNiBRq/271PlxjZQ1dAID+2FDAkbhjYcERXPDoZ6hp6w06FCIKKTMjO2MAHlZK7RKR0wEUiMgq7bGXlFLPexceLSmsAQAcauzCuWf8WcDREJGRAEboGydypVQdgDrt504RKQFwrteBERFFiUhwZVtqIxeRHACXA8jTNj0oIkUiMkdExuk8Z5KI5ItIflNTk6NgiYhoNNOJXEROA7AIwK+UUh0A3gBwEYDLEK+xv5DueUqpmUqpXKVU7vjx/k4kQ0Q0FphK5CJyMuJJfL5S6mMAUEo1KKWGlFLDAN4GcJV3YRIRkR4zvVYEwGwAJUqpF5O2T0ja7XYAxe6HZ12inSqIGw5EREEw02vlGwDuArBXRAq1bb8BMFFELgOgAFQC+L+eRGhR4n5DtuRxXpCIyIiZXiubcTw/JvvM/XBIj+4d8TSJvqs/hq+cfCJOOCHA2+hE5JusG9k5Vuil6O7+GP7miRV49vMSX+MhouAwkWeZxOokSwprA46EiPzCRE5EFHFM5ERELlIBdLXI2kSebsbAVEPDCt+avg6fFrEZgoicEd07V97LukQuWvcOM932ugdiqGrpwaOL9noclX1BXN2JKFoil8iVUqhq6Q46DN8FebWn8NhV3Yr23sGgw6CQiVwin59XjW9NX4+CqtagQwklDiDKXrGhYfz49a24Z+6OoEOhkIlcIt9VHU/gXzRbq5XXtffi2mfXoLqlx4uwQifIKTXJG8PaRXpvTXuwgVDoRC6R2/XxrhrUtvfhg53VQYcyStGRNqwpabD1XFbAiWjMJHIvDcSG0dhpfxHkH83Ygnvn5Vt6DmvcRJTARO6Cf/lgN656Zo2vZbItnIgSsi6RB1FTXb6v3rNjGyVsVsyJyMw0tpGUbTVWMxeozWXN6B0c8j4YItIVysWXoybsbcdDwwrDSuHkE93/MnTn7DzjnYjIG1FZfDkbeHm1fGJJMerbM9/0/PEbW3HxlM+9C0LDAUREY8eYSeTpaupuJ/V526rwyKKijPvsOdzmbqFENOZlVSKvbunB4aO91p/oYuXVzGRdRERuyqpEfv30db6U09Yz4Es5RERmZEUib+0eGFUT9mrWwCWFNbjsqVUoOuJPE4lRBZ/1fyKKfCKvbO7G5U+vwtwtlSO2e9XCsaW8GQBQUtfhTQE6Ult/wt47h4j8E/lEXn00PgnWutLGgCMxtrDgSNAhEFEWyrp+5GZNWbwX8/Pcn0Ar0zeBf/toj+vlERFFvkZuVaLtfEQSV8B72yrx5oZDwQTlATa9EAUjiPtWkU/kP5tjbpJ9owEyjy3Zh2mfH0BsaNiNsEJtIDaMb0xbi5UezhFDNNYEWXeKfCJ32+DQyOvp/toO1LbZ6JvugNd90Vu6+1HT1ovHl+zztBwi8kfWJnIFoLGjD99/ZZPhsPlMbn1lE66dtta9wAzsq23HBY9+duzmLRdfJiIjWZPI01ViP9hxGPtqO7Agr8r/gGzapa1FOmrFIJ3vbYt316BnIOZxVEQUZoaJXETOE5F1IrJfRPaJyEPa9jNFZJWIlGn/j/M+XPOcNk/kTl2NKYv3Zji+o8O76vkVB4MOgWiEiTO347V15UGHMWaYqZHHADyslLoUwNUAHhCRSwFMBrBGKXUxgDXa79GUprbb3NWftnui0U1Tt5tCzFwwuvoHdR/rGxxCXbu/bfxE2ypaMH1FadBhjBmGiVwpVaeU2qX93AmgBMC5AG4DME/bbR6Av/cqSKfe3Z7UtOJSnv3I58E9qRcQo+6FiYfvn78L1zzrXxs/AOyubkVzV7+vZRKNZZbayEUkB8DlAPIAnKOUqtMeqgdwjs5zJolIvojkNzU1OQjVvraeQWw91OzqMQuqWtHUGf5ktfaA/yNeb399K37wymbfyyUaq0wnchE5DcAiAL9SSo2YaETFG6TT1nWVUjOVUrlKqdzx48c7CtaJ/kFz/cOtDKQZGg6uoTy5yeXD/CP4q8fsL1ax7kAjBl3uP1/fYb+nEBFZYyqRi8jJiCfx+Uqpj7XNDSIyQXt8AoBAJztJbZsO0b1IWwZiw3hiSTFaTU6Z22fyQpVq66Fm3PPOTry4ijdMybnfry7D7urWoMMYcwznWhERATAbQIlS6sWkh5YCuBvANO3/JZ5E6EAk+2BrXwk+LapDz8BQ6mZXKSgc7Y5fKKpbetwvgMacl1YfxEurx3alIIjFZczUyL8B4C4A3xWRQu3frYgn8BtFpAzADdrvoRFU90C3yvWy2YbreVImB+o7sOFgMPezokwCnODIsEaulNoM/WkEvuduOMY+3lXjd5GRNWfzF0GHQBF088ubAACV074fcCRkVmRHds7aVIGcycssPy+CjS22iAie+nR/0GEQkQ8im8gP1HeO+N2wSYOtCUSUpSKbyIPCeb6JKGyyJpGPrpGbb0SJYm7mBYWIErImkacTpomtrEoN3eu8HYVRqhTn1/t6eXE9rnh6FfpjQ8Y7U6CyOpGb5ebnQu9DZnbukWMJW+c4nxbV4v75BZbjMrKj8qjrxyR3Jb6FxXwaUfz0p/txtHuAF/kIyNrFl/US6vCwQmnKjVI/3DZjiyvHeXDBbgDAjZemndqGxoihYYUTT2D7GsVFpkY+NKwwa1OF7ecn3vKzNn+Bv3t5o+5+TlYT0tPSFV9aLSySR7xycFA0DUe53ZBcF5lEvnRPDaYuK7H9fLM3B7/9/HrbZehp7dGfLzwIiW8HzAUUZc1d/Vjo4XTSZQ2dKKgyP29MkFWiyDStdPdbu+Hy3IpSFNe0exRNtDWaaPNs6uxH4eE2NuFQaN33XgHyq1pxzUVn4dwz/sz149/4UvybexRGuEamRm4kdYKsVfsbUJfUTNJrc3ZA2/E4rO4GPeHXXbPz8Mt389E3yB4LFE6JCknM5SmYoyhrErmRgZg7L3ZyE0265ppEAnbabDE4NPIAXk3IU1yb/ltLZUs3ADa/EEVB1iTy7RXudZ87fNTclK5eJDk/B/o0dvbjjfWH/CuQMmrs7MMtv9+E2hDdGKdoiEwi97NieN1z62w/N11bvhfJeaz0NlFK4cP8wxm/UX2487CthTH6Y0PImbwML4dk/uyFBUdQUteBd7dVGe9MlCQyiTyM3/EVFGasLRuxbW+aG6xBpFynF4+wnO5PiurwyMIivLauXHefRxYV4ZU1ZbqP60lcdOdtrbQbXmDC8vqQsR++uhl3zsrztIzI9Frxkt2cV1jdhudXpq/NZcvnzOwFYduhFrT3DuDmv5ngavnt2lJ3Ld32Rhe+vr4cp51yEn52TY6LURGZl65y57Yxk8hL6jp0H7Nbu8k0VNrr5Z687tVi9egT394OIHxdtZ5bXgoATOTkqbKGTqwvDW7Z4ug0rXiosz9m63k9A/aeR0Tea+zs820h6Btf2oiOvuDyQWQSeRibKv5j0V7fyuK0te4LYpFc8s8tL2/C7a9v9b3cIN5WkWlaidJnLmfyMtx19fnHfk/XB7ylqx9nnXbKqO2h6Y0SgvPd2NGHpXtqAQBFR6y1MxbXtOMHr242tW+Qi+Ym9AzEsLSwNugwskpL90DQIfgmMjXyqHlve+YuZFdOXY01JQ0+RWNfkDnuztl52FkZ/2pcdKQd3f0xVDZ3Z+zBkvCn3caLdIfgWnXM75buH7V8IZkTpUqeV5jILbOe2fSescvD9rsQVDIdq0uZiXJwaBg/nZWH6StKcdTF2pYbp2pJYQ06++xPjlbbzkFAVmXDe9wtTOQR1Tvg7fwSVnrFlDd2eRjJSG7eYHarJldS14GH/lCIRxYWuXI8M+c+6Ll4KFwik8ijfGOqvsP9Oc79+iCbabP/53k7XS1zw8EmfLLH2/bi/bUdqNamYnBas+sZiA8s8uJ1zlb7azuQM3kZ9hxuCzqUrBCdm51BB+DA8uL6oEOwzMp10+5r09zVj4HYMP5HyhSkd8/ZAQA4/cvevT1vfWWTZ8cmY2sPxO8Prdxfj6+dd0bA0URfZGrkQXMyVeY7FoaA693IC6o5UK+2uq60EXO3fAEAGLQ5s2Tu1NW4dtpau6ERkYaJ3KTpK0o9L0MpFaol4TK5Z+5O/O6T/QA8/LYU5a9hNGZVm5w91U2RSeRBr+S9vaLF8zJ++W6BZ8cO8z2GVfsbsLy4LugwAuVGX/b+2NCIb45Dwwr9MS4M4tTTn+63tP+Lqw562iMtHcNELiJzRKRRRIqTtj0pIjUiUqj9u9XbMIHXQzJvdnOXdxeU1Rn6lVc0dXtWrhE358eubetFXUpXu1++m4/73t9l+FzvrkX6SXRrebOJ+enDcZG85LfLccvvj7f93z+/AJf8dnmAEYWLUgqby5oxnGGOpHRmb/7CclmVzf5+Xs3UyN8BcHOa7S8ppS7T/n3mbljh09Ufw+DQMFbtd28Qj1LArz8sNLXvxrImS8d2OkI08VZfU9KAa6etxdoDDZi1qQLtPYOOJge6dtpaXPNsuNrFM1WGfzIrz/T89E7OePI3zrc2VOA/P7O30HhZUlfQFfvCP+DMTyv2NeDO2XmYG8Fpi40YdgtQSm0UkRzvQwm3Q03d+MU77nazA4CPdxmPQARGf/V+7E/FOnvGtfY4GzCTaIrZow2Nn7ulEpvKmrGruhWf7Q1/L5yoDRY5lNIXf+bGCvzm1r/W3T/ELWWhlfgmaHYFsChx0kb+oIgUaU0v4/R2EpFJIpIvIvlNTdZqlWGzqaw56BCOMfp22OnyTGz92uLVbh83EzdzVbfNGS7DJBv+Bi/wmmY/kb8B4CIAlwGoA/CC3o5KqZlKqVylVO748eNtFdbea3/oc7aIWAXTFi9vyD70B3NNWEEZMNG99YEFxvcRwkwphXWljZbbqPX4/Zk4UN8R2qmrbSVypVSDUmpIKTUM4G0AV7kb1kg/0RYtIP8EUctx6fOdVqaFRaJykdxfO/JvaO0eiNRCzcv21uGeuTsztlEPDg27lugT2noGMO3zAxhycNy+wSHc/PImPLhgt6n9/W7as5XIRSR5Pa/bAWRusHVoX63+h5C8FZUkB8RrfHstTnfrTrm+FwkAuPrZNZEaUFWvTYJW06p/8bl4yue4e+4OV8uduqwEb2445GiEdeIb084vjroVlqvMdD/8AMA2AJeIyBERuRfAcyKyV0SKAHwHwL96HCcBKKjyr2+qleTkV7I3Cum97VX44YzN2GSxh49b/J7XvN/miNqwc/te1IB2njItzRh1ZnqtTEyzebYHsZCBKYv9W5EowWlu6uwbRHFNB6656CxX4sn0UUzM513VYq1XQlh7uHT2DeL0L58cdBgUAZEZ2UnOVTZ3m2p/dLN71oMLdmPi29vRamP+8Ew3Pzv7BpEzeRlWu9Cvv7GzHzmTl5lasMJP//6RO9PiUvZjIg+Qn1/0DtR34tvPr8dbGysM91174PiAn0Qu3VFpr20wcZPRTK8MKw42xPtdv7HB+YjfxN84fUUpBl2OM2FRwRHkTF6GBgtT3da09aKps9/RTbqwe23dIeRMXhbqKSTs8HvJRibyALVYGO6/dE+to6XAEhP5FFTpJ+SpaeaUCPLj1T1gf56QTO3VmUbnPrMsPqKyvXcQbSmDqpwk+YUFRwCMHviTSUtXP/72mdW+TNgWND/yuBtFpDtGcY3/N9hTMZEHyM3lytwwK82cElsPGU8W5uZnsLHTXI01P80FaUFe9bGfM9XwEtPvppOY7Ohrv1uJy55adWz7gfoOXDzl82OTe+344iiKjsQXRUhXklIK722vcrT8W7P2/nhzwyFsLmsOxSLRbtD7M37wanjniM905sMwYykTeaDC/8E0WsHl1TVlto6bO3V12u1GNbPE429tMG4iclOR1q1xdUm82el/v7UNP5qxZcQ+ya/m9oqjeOxPxXh8yT5Xyr9zdp4rxwmz4hp2M7aLiZwsGUjp8vbCqoO2juPlLJJONHa4E1ffYLxZyOmcN2QsjO3rkRgQRNGTeK8napR25RkMiPjdJ+7UQNPx4wNrtSmksrnb0lz56f6Csgb79z7SlqFUxhWtatp68fO5O9AV0Nwtbr2MQTc19Q0O4frn1mFrefBzMEVmzc5slGn+cff5U2uZu6XSdqkt3cHX0q0mh28/v97kcfUfe8PlufYX7KjGlMX6g61fWFmK9aVNWF5cj3UHGvHdv/rv+Icr/8LVGBKqW3rw1bO+4smxg/ZFczeqj/Zg3raqoENhjZycM5P6zKxWc9Uza9wJyIGu/ljkpzldWlhret9le+vw8Ed7PIvl+unm5nI3q7imHRc+umzU4iRjHRM5+eKf3tzmeLWaNguzYL69yfqqLgnffWH96I3aV4tEN0IjyuNvQMnNE7fN2HysB022e397FYYVsL40mGkYwtgeDzCRjxlm339ur42aKNeNLlrvb68yNTLV6UdtcMjeEfKrWo9N2pX4u/1oxd1zpN3yupJBStfMFPaelUG3xxthIh8jXl1rbvj53z6z2tVah9XeKZk+LnO3VKLTzA26NPF39g2O6nHjhR/O2Dzi99QEkPbU6v3R4az8+W5reXMo+mqHGRP5GLE/w3zcqbIxf3T0xXDvPPeX6jMrU4XO7+HcUfOTWXn4xrS1jgZXZTsmcvJMUN3b9CSmR/VlOLgHZYRtJLBZR7sHUNF0fGqC7RXGo4XT+c7zG5wF4vKLEqbWFiZyGqXQYDSnWdebXH0+m3n1WU+9SCbnKLMJJlNf80z++rHluPnljab3//b0dfjuC8eTsN2FYtwaRBamBOwWJnIaZYmF7muZRLUGmY6XvVDsJBY3ugw+ZfMGae/g0LEJ3DaVNWHWpszTJXRoC3aHtMOHJWb/BL9vjnJAEJEJ/7HI/KIezV39iA2PrO0mpqJNd0HQ/chnyAWZZlE02+be42B2yYS7ZseXZfvn6y50fCy7jJKrG23rYa/EM5ETuSx36mqceeqXRmzbXmF9PncrvWx6B50nZSdeXFmKSd+6KOM+GW/4mqzB/mHn4ePPMfUMYJ3W53yPy+u5Wp0Qrbs/hlNP8SblsmmFwsWFqk9bT/C9GxLNSqn56eEP92DlPvuLAOsJeoHyV9aW4/ksnjc9Xa1/h8WFmIs8XBiciZwcq203v+qNH15cbW9GRm+MzOSNnf2Y9F7ByD3C/r3dJKMpGNxsIw+qvT2sLxUTOVEAfvun423ubvcjN7owBNVv/aDLszz66fWQreeaiomcKADvb6823ilJGGrtsaFhg55ImYO0s/aoFz2f7FzIXnd5hkq3MZFT1olKN7cnlhTjfz7ubCIxsw4f7cGRVmezOj6+dB+ueHqV7uNGFxs7L8t97xcY7+Qjs3/DW2kWBffyYsxeK0QBScxj7Udt+zptcFbltO9n3K8jQ1e9ZUV1rsaUSd/gEL588olo6Ajm/ovdwVIJft98Zo2cyENmkrSZfax8y7B7Yfj1h4X4X0+utPdkE6xMxnbHzO2exWHm/Kw54GwlrbTlun7E45jIKVT8uBHn9Vzhybz6azIlRbvn8ONdNXbD0cp1j1vTRNjlxbzjXo72ZCKnMSV58iZyl5M81R6Cvv9RxkROY0ry5E12Wa2tHTK8eISgS0rAvvZU+iYdM6c6Kje3vWSYyEVkjog0ikhx0rYzRWSViJRp/4/zNkyi8JhlYRm5gqpWfM/g4uH2N+5M3fy8XKrMqEnH9ZK14vYcbvNgndX0f4uV07e+tBFbypuPH9HD67WZGvk7AG5O2TYZwBql1MUA1mi/E0WCnXlPkm0sM79eZEsAM0BuyzDf99sGMxVmEsR6lWbuZzz80Z5jvXLC5Odzd+Kns/J8KcswkSulNgJIfeffBmCe9vM8AH/vclxE5JLqluO11Q0H01+EegaMFwFxmsYbO9xbD9bPG9ZuCWOvlXOUUolOpfUAztHbUUQmiUi+iOQ3NQWz8jVRmHndQr72QIPhPv2DmftNv7utEp19zlZ8yvRNwaq3NlQgZnORbADoHRhCeaO/UwYE3bSSkYp/39I9o0qpmUqpXKVU7vjx450WR1kuDEPR/ebn3yyQtOXN2ZK53d/MlK1+/h3VR3tQ7aBd/IEFu3DDixt1J/ry4m/JszhbohV2E3mDiEwAAO1/93vPE9k0Fi4Gduuieufmi+Zu27EcO7bjI4zmRbP8oaYubD0Uvwk57GwApyX5la2eHdtuIl8K4G7t57sBLHEnHCLnvL4nV9YQrb7o0WtNdm7pHv3lCteXNqFPa0qKYlt7Oma6H34AYBuAS0TkiIjcC2AagBtFpAzADdrvRGNCvcvzf3g9mrWly7jnjBujDv1ep1LP0LDCv3yw29S+erMr6v0lThK/l2fHcNIspdREnYe+53IsRGNSfpV3X7kBYEbSXNp6yTZd18Katl4AwNsb03dZvP31LS5El5mdb1dWukme4OPFJ9Q3O4nImZI66zPl2W0+stMXPDFLY6rd1cHOh6InOxpLrGEiJ4oguyvDd/U760IYdg0dfZYWova3Nci7wjgfOYVKOFpZw697wHyySqZ3fsPSvp2sqdP6AKKv/+caXHm++RlD9O5PhPF8ZMIaOUVaa4//Q+CjzMsE5fahf/1h4bF2eisKHNxzONo9gHlbKwOZjsAJJnIKlSqLgzwqmpz3fx5rFhYcGbXNjcTldu+bPgtNJHb2B0ZffP71j4V4Yuk+lNS5P+qTNztpzPjx61uDDiGrRa2maYXThTEAoE37hhfzYKRQGOdaIaII0mtacacfueNDjGD1mvObxXstlxGtlnB9TOREY0i2JC7XpJyQPUfavSuKTStENNZ4PVDKieEMi3cEgYmciAhAZdLEYUaV54W7Rt8wDhITORG5wkrLwRfNbi/NZk9yTxszXR0T7fatNlZ+8nJOHSZyInKFlTbgRSGr0fqBbeRE5IqIDVgMTsROFBM5EbkiasPagTE0HzkRkRnRS+PW9cfsDxRi0woRkceS82xVS7SmfmAiJxpD9HpOfJJhaTSznNRWw2ZJYebzYad2zV4rRBR672ytDDqEcGPTChGRB7LjXicTOdGYMhbuSI5BTOREY8jgUPa0Y7suwhc5JnKiMSSsCyaTM0zkREQWfJR/2NbzuLAEEZEHFIAP8w/jve1Vpp/z7wuLPO1KaMdJQQdARBSkRxYWAQAW/PLrnpbj5RQGrJETEUUcEzkRkUXDIVvEmomciAhA/6D5rpkz1pVbPj5vdhIReSC5Yn3POztNP6+zL2a5LC9nP3R0s1NEKgF0AhgCEFNK5boRFBERmedGr5XvKKWaXTgOEZGvPi1yPuujWWxaISLywNRlJUGH4AqniVwBWCkiBSIyKd0OIjJJRPJFJL+pqclhcURElMppIv+mUuoKALcAeEBErk/dQSk1UymVq5TKHT9+vMPiiIiiKbQDgpRSNdr/jQAWA7jKjaCIiLJNKNvIReRUETk98TOAmwAUuxUYERGZ46TXyjkAFmtfF04CsEAptdyVqIiIyDTbiVwpVQHgay7GQkSUvbhmJxER6WEiJyLygZdzmDORExH5wMu5VpjIiYgijomciCjimMiJiHwQygFBRERknpdrCjGRExFFHBM5EZEP2LRCRBRx7H5IRES6mMiJiCKOiZyIyAccok9ERLqYyImIfMCbnUREEcdETkREupjIiYgijomciCjimMiJiCKOiZyIyBfsR05ERDqYyImIIo6JnIjIF94tLcFETkQUcZFI5C//n8uCDoGIyKExfrPzyvPHBR0CEZEjBVVHPTt2JBL5eWd+JegQiIgcqWjq9uzYjhK5iNwsIqUiUi4ik90Kiogo28SGQ3izU0ROBPAagFsAXApgoohc6lZgRERkjpMa+VUAypVSFUqpAQB/AHCbO2GNNv70U7w6NBFRpDlJ5OcCOJz0+xFt2wgiMklE8kUkv6mpyXZhO6fcgMX3X2v7+UREQXr/3q97duyTPDuyRik1E8BMAMjNzXXUSHT5V8ehctr3XYmLiChbOKmR1wA4L+n3v9C2ERGRj5wk8p0ALhaRC0TkSwDuALDUnbCIiMgs200rSqmYiDwIYAWAEwHMUUrtcy0yIiIyxVEbuVLqMwCfuRQLERHZEImRnUREpI+JnIgo4pjIiYgijomciCjiRCnvJnIZVZhIE4Aqm08/G0Czi+G4hXFZE8a4whgTwLisCmNcbsV0vlJqvN6DviZyJ0QkXymVG3QcqRiXNWGMK4wxAYzLqjDG5VdMbFohIoo4JnIiooiLUiKfGXQAOhiXNWGMK4wxAYzLqjDG5UtMkWkjJyKi9KJUIyciojSYyImIok4pFfp/AG4GUAqgHMBkD45/HoB1APYD2AfgIW37k4jPsV6o/bs16TmPavGUAvg7o1gBXAAgT9v+RwBfMhlbJYC9Wvn52rYzAawCUKb9P07bLgBe0cooAnBF0nHu1vYvA3B30oJinmoAAAUQSURBVPYrteOXa88VEzFdknROCgF0APhVEOcLwBwAjQCKk7Z5fn70ysgQ03QAB7RyFwM4Q9ueA6A36Zy9abfsTH9fhrg8f80AnKL9Xq49nmMirj8mxVQJoNDP8wX9nBDoe0v3M+l2UnT7H+JT5B4CcCGALwHYA+BSl8uYkDjxAE4HcBDxBaWfBPBvafa/VIvjFO3Ne0iLUzdWAB8CuEP7+U0A/89kbJUAzk7Z9lziAwRgMoD/0n6+FcDn2pvqagB5SW+MCu3/cdrPiTfgDm1f0Z57i43Xpx7A+UGcLwDXA7gCI5OA5+dHr4wMMd0E4CTt5/9Kiikneb+Uv81S2Xp/n0Fcnr9mAO6HlnARX7fgj0ZxpTz+AoDH/Txf0M8Jgb63dD+HdpOfX/8AXANgRdLvjwJ41OMylwC4McObfEQMiM/Jfo1erNoL1YzjH+QR+xnEUonRibwUwISkN1yp9vNbACam7gdgIoC3kra/pW2bAOBA0vYR+5mM7yYAW7SfAzlfSPlw+3F+9MrQiynlsdsBzM+0n52y9f4+g3Pl+WuWeK7280nafpIprqTtgvjawBcHcb7S5ITA31vp/kWhjdzUIs9uEZEcAJcj/hUQAB4UkSIRmSMi4wxi0tt+FoA2pVQsZbsZCsBKESkQkUnatnOUUnXaz/UAzrEZ17naz6nbrbgDwAdJvwd9vgB/zo9eGWb8AvEaWMIFIrJbRDaIyHVJsVot2+5nxevX7NhztMfbtf3NuA5Ag1KqLGmbr+crJSeE8r0VhUTuGxE5DcAiAL9SSnUAeAPARQAuA1CH+Fc8v31TKXUFgFsAPCAi1yc/qOKXbRVAXNCW+PsRgI+0TWE4XyP4cX6slCEiUwDEAMzXNtUB+KpS6nIAvwawQET+mxdl6wjda5ZiIkZWFHw9X2lygu1j2WG2jCgkcl8WeRaRkxF/weYrpT4GAKVUg1JqSCk1DOBtAFcZxKS3vQXAGSJyUsp2Q0qpGu3/RsRvkl0FoEFEJmhxT0D8RpGduGq0n1O3m3ULgF1KqQYtxsDPl8aP86NXhi4R+TmAHwD4qfYBhVKqXynVov1cgHj781/aLNvyZ8Wn1+zYc7TH/1zbPyNt3x8jfuMzEa9v5ytdTrBxLF/eW1FI5J4v8iwiAmA2gBKl1ItJ2yck7XY7gGLt56UA7hCRU0TkAgAXI37jIm2s2od2HYB/1J5/N+JtbkZxnSoipyd+Rrw9ulgr/+40x1oK4GcSdzWAdu0r2goAN4nIOO2r802It1/WAegQkau1c/AzM3ElGVFbCvp8JfHj/OiVkZaI3AzgEQA/Ukr1JG0fLyInaj9fqJ2bCptl6/19meLy4zVLjvcfAaxNXMgM3IB4O/KxJgi/zpdeTrBxLM/fWwDCf7NTe71vRfyu8SEAUzw4/jcR//pShKRuWADeQ7x7UJF2cickPWeKFk8pknp66MWK+F3+HYh3NfoIwCkm4roQ8V4BexDvAjVF234WgDWId09aDeBMdfzG0Gta2XsB5CYd6xda2eUA7knanov4h/cQgBkw0f1Qe96piNeq/jxpm+/nC/ELSR2AQcTbGe/14/zolZEhpnLE20pHdJsD8A/aa1sIYBeAH9otO9PflyEuz18zAF/Wfi/XHr/QKC5t+zsA7kvZ15fzBf2cEOh7S+8fh+gTEUVcFJpWiIgoAyZyIqKIYyInIoo4JnIioohjIiciijgmciKiiGMiJyKKuP8PsdJcMAD7edMAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(losses_ru)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "_is_relevant = [.0, 1.]\n",
    "_is_not_relevant = [1., 0.]\n",
    "\n",
    "_logger = logging.getLogger(__name__)\n",
    "\n",
    "def test(data, model):\n",
    "    true_positives = 0\n",
    "    false_positives = 0\n",
    "    true_negatives = 0\n",
    "    false_negatives = 0\n",
    "    for item in data:\n",
    "        expected = item['answer']\n",
    "        node_vectors = item['graph']['vectors']\n",
    "        item_vector = item['item_vector']\n",
    "        question_vectors = item['question_vectors']\n",
    "        question_mask = item['question_mask']\n",
    "#        try:\n",
    "        prediction = model.predict(node_vectors, item_vector, question_vectors, question_mask)\n",
    "        if prediction == expected and expected == _is_relevant:\n",
    "            true_positives += 1\n",
    "        if prediction == expected and expected == _is_not_relevant:\n",
    "            true_negatives += 1\n",
    "        if prediction != expected and expected == _is_relevant:\n",
    "            false_negatives += 1\n",
    "        if prediction != expected and expected == _is_not_relevant:\n",
    "            false_positives += 1\n",
    "#        except Exception as e:\n",
    "#            print('Exception caught during training: ' + str(e))\n",
    "    try:\n",
    "        precision = true_positives / (true_positives + false_positives)\n",
    "        recall = true_positives / (true_positives + false_negatives)\n",
    "        f1 = 2 * 1 / (1 / precision + 1 / recall)\n",
    "        print('precision', precision)\n",
    "        print('recall', recall)\n",
    "        print('f1', f1)\n",
    "    except:\n",
    "        print('Cannot compute precision and recall.')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test RNN trained on 10 node subgraphs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "precision 0.5052192066805845\n",
      "recall 0.24719101123595505\n",
      "f1 0.33196159122085045\n"
     ]
    }
   ],
   "source": [
    "test(test_set,RNN_with_attention)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test RNN trained on 100 node subgraphs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "precision 0.9183457051961824\n",
      "recall 0.8774062816616008\n",
      "f1 0.8974093264248705\n"
     ]
    }
   ],
   "source": [
    "# try testing the one trained on 100 on the 10 subgraph set\n",
    "\n",
    "test(test_set,RNN_with_attention_100)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test RNN trained on 400 node subgraphs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Tested on 400 nodes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "precision 0.8771929824561403\n",
      "recall 0.9155645981688708\n",
      "f1 0.8959681433549029\n"
     ]
    }
   ],
   "source": [
    "test(test_set,RNN_with_attention_400)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Tested on 100 nodes "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "precision 0.8757281553398059\n",
      "recall 0.9222903885480572\n",
      "f1 0.898406374501992\n"
     ]
    }
   ],
   "source": [
    "test(test_set,RNN_with_attention_400)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Tested on 10 nodes "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "precision 0.8726738491674828\n",
      "recall 0.9101123595505618\n",
      "f1 0.891\n"
     ]
    }
   ],
   "source": [
    "# pickle file, saved from running the make_training_set_wikidata notebook\n",
    "   # name has model, number of entities and number of nodes\n",
    "\n",
    "current_test_set =  'test_data_rnn_1944_10.pickle'\n",
    "with open(current_test_set, 'rb') as handle:\n",
    "    test_set_10 = pickle.load(handle)\n",
    "    \n",
    "test(test_set_10,RNN_with_attention_400)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}