FFN_digit_recognition.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "rGKB_rr4deRk"
      },
      "outputs": [],
      "source": [
        "# Bringing data [Motaz]\n",
        "# Hyper parameter optamization [Omar]\n",
        "# Save model in TensorFlow Lite [Omar]\n",
        "# Regularization (Drop Out) [Motaz]\n",
        "# Regularization (early stopping) [Motaz]\n",
        "# different optimization algorithms [Motaz]\n",
        "# Figuring optamization methods [Meqdad]\n",
        "# Mobile Application template [Meqdad]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "74Od7wGebLXc",
        "outputId": "1de7626e-f478-4ebd-e461-7b2b10ab64da"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2.8.0\n"
          ]
        }
      ],
      "source": [
        "# TensorFlow and tf.keras\n",
        "import tensorflow as tf\n",
        "import keras_tuner as kt\n",
        "from tensorflow import keras\n",
        "\n",
        "from keras.callbacks import EarlyStopping\n",
        "from keras.layers.core import Dropout\n",
        "\n",
        "# Helper libraries\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "import random\n",
        "\n",
        "print(tf.__version__)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eSB0NKO5fCyt"
      },
      "source": [
        "The MNIST database contains 60,000 training images and 10,000 testing images of handwritten digits. We will use the dataset to train our digit classification model.\n",
        "\n",
        "Each image in the MNIST dataset is a 28x28 grayscale image containing a digit from 0 to 9, and a label identifying which digit is in the image."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 21,
      "metadata": {
        "id": "OXaCUuqmcIFM"
      },
      "outputs": [],
      "source": [
        "# Keras provides a handy API to download the MNIST dataset, and split them into\n",
        "# \"train\" dataset and \"test\" dataset.\n",
        "mnist = keras.datasets.mnist\n",
        "(train_images, train_labels), (test_images, test_labels) = mnist.load_data()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 589
        },
        "id": "Rp2QfrFlfJvR",
        "outputId": "12898408-446c-4f23-8afd-8c21582dddcf"
      },
      "outputs": [
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAj0AAAI8CAYAAAAazRqkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdebxV8/7H8c9XhKhQyVyIUlHmMlQ/yRAyUyq3zMNVhkKGZAqpSxIh81A3ikyXXMmsq6hbGiSKEkUTSoPW7w/5+HzXPfvYZ5+99zp7f1/Px+M+7nu1vnvtL6t1ztf6Ti6KIgEAACh2GyRdAQAAgHyg0QMAAIJAowcAAASBRg8AAAgCjR4AABAEGj0AACAIG5alsHOO+e0JiKLIZfua3MvE/BBFUa1sX5T7mQyezaKS9WeTe5mYlPeSNz1Afs1NugIASsSzWTxS3ksaPQAAIAg0egAAQBBo9AAAgCDQ6AEAAEGg0QMAAIJAowcAAASBRg8AAAgCjR4AABAEGj0AACAINHoAAEAQaPQAAIAglGnDUaCi2Xfffb3jv//975rPPPNMzU888YRXbtCgQZo/+eSTHNUOAFCR8KYHAAAEgUYPAAAIAo0eAAAQBBdFUfqFnUu/cAIqVarkHVevXj2tz9lxIFWqVNFcv359r9zFF1+suX///po7dOjglfv1118133777d65G2+8Ma06WVEUuTJ/6C9U9HtZmqZNm2oeO3asd65atWppXWPZsmWaa9SokZ2KpWdiFEX7ZfuihXw/s61169aan376ae9cy5YtNc+cObPc38WzWX7XXXedd2x/Rm6wwZ//Xd6qVSuv3Ntvv53tqmT92QztXlYgKe8lb3oAAEAQaPQAAIAgVNgp6zvttJPmypUre+cOOuggzYcccojmLbbYwit38sknl6sO8+bN847vuecezSeeeKLmn376ySs3efJkzTl4BRukAw44QPPIkSM1x7swbXetvS+rV6/2ytkurebNm2ueOHGiVy7+uWLSokUL79j+O3n++efzXZ2s2X///TVPmDAhwZoglS5dumi++uqrvXPr1q0r8TNlGYoBpMKbHgAAEAQaPQAAIAgVpntr77339o7ffPNNzenOwsoG+2o1Pqvgl19+0fzMM89o/vbbb71yS5Ys0ZyNGSKhsDPn9tlnH+/cU089pXnbbbdN63qzZs3S3K9fP+/c8OHDNb/33nuar7/+eq9c37590/quQhSfDbPbbrtpLrTuLTvLZ+edd9Zsu8lFRJzL+mQrZKBOnTqaN9544wRrErYDDzzQO+7cubNm2/3dqFGjlNfo0aOH5vjvwkMPPVTzk08+qXn8+PFlr2yW8KYHAAAEgUYPAAAIAo0eAAAQhAozpmfu3Lne8Y8//qg5G2N6bB/i0qVLvXP/93//p9lOUbZ9kMi9Bx54QHN8letM2HFBm2++uXfOLiVgx7bsueee5f7eQmF3oRcR+fDDDxOqSfnZcV7nnnuuZjsWTERkxowZeasTfIcffrjmSy65JGU5e4+OPfZYzd9//31uKhaY008/XfPAgQO9czVr1tRsx7+NGzfOK1erVi3Nd955Z8rvstew127fvn36Fc4y3vQAAIAg0OgBAABBqDDdW4sXL/aOe/bsqdm+4hQR+fTTTzXbVZLjJk2apLlNmzaa7dRzEX86Xvfu3dOsMbJh33331XzMMcdoLm1qse2aevnll71z9lXrggULNNu/MyL+sgKHHXZYWt9bbOw070I3dOjQEv/cLluA/LKr5YuIPPbYY5pLG7Jgn+H4sAekZ8MN/V/tdpXyhx56SLNdJkRE5J133tF88803a7bLeoj4ywyMGDFC8xFHHJGyThVldfTi+akHAABQCho9AAAgCDR6AABAECrMmJ64F154QfPYsWO9c3b37CZNmmg+++yzvXIDBgzQHB/HY3322WeazzvvvLJXFmlr2rSpd/zGG29orlatmub4jsr/+te/NNvp7C1btvTK2a1D7DiPRYsWeeUmT56s2W49YscVifjT3j/55BMpdHvttZfm2rVrJ1iT7Eo1RsT+/UJ+/e1vf/OOU20fE58O/cQTT+SqSsHo1KmTd5xqzFv8+bDT2ZcvX57y+rZcaeN45s2bp/nxxx9PWS6feNMDAACCQKMHAAAEocJ2b1mlvWZbtmxZynPnnHOOZrurtu3OQO7tvvvumu1SBCJ+t8QPP/yg2U43F/Ffjf7888+aX3nlFa9c/LisNt10U+/4iiuu0NyxY8dyXbsiaNu2reb4P2shiXfN2Z3Vrfnz5+ejOljPrrp71llneefsz127Kv6tt96a+4oF4JZbbtHcq1cv75wdLnDfffdptsMBREr/XWtde+21aZXr1q2b5vgQg6TwpgcAAASBRg8AAAhCQXRvlaZPnz6a7eq+Iv7MHrvZ3ZgxY3Jer5DZ1TpFRPr376/Zdq+I+DPx7AaY8dU7k+qK2WmnnRL53lypX79+ynN2FmNFZ/9OifjdXZ9//rlm+/cLuVG3bl3NI0eOTOszgwYN0hyfnYv09O7d2zu2XVp242wRkddff13zVVddpXnlypUpr7/JJptojs/Qsj8X7Sr2totNRGT06NEpr58U3vQAAIAg0OgBAABBoNEDAACCUPBjeuxKy+eee653zq6ga3eWfeutt7xydvzI4MGDNcdXBUZ67CrGIv87jsc6/vjjNdvd05F/H3/8cdJV8FblFhE56qijNNtVZktbBdbuDm2nRiM37D2yK37Hvfnmm5oHDhyY0zoVqy222ELzRRdd5J2zv6/sGB4RkRNOOCGt69erV0/z008/rTk+XtZ67rnnNPfr1y+t70kSb3oAAEAQaPQAAIAgFHz3ljV79mzvuEuXLpofffRRzZ07d/bK2ePNNttMc3zju/gqwSiZ3ehVxJ/SGO/CqghdWhts8GfbP75at617sdtqq63K/Bm74a+I/++ydevWmnfYYQevXOXKlTXbla7t50X8KbXjx4/XvGrVKq/chhv++aNs4sSJadUdmYl3ldx+++0llnvvvfe8Y7sBaWkr6SM1+9zY1a/j7ErIIiJbb7215q5du2pu166dV65x48aaN998c83xoR72+KmnntJc2sbeFQVvegAAQBBo9AAAgCAUVfdW3PPPP6/5iy++0BzvfrGv4fv27au5Tp06Xjm7MR4bGfqOPfZYzU2bNvXO2VehL774Yt7qlC7bpRV/jTtp0qR8VyenbHdR/J91yJAhmq+55pq0rhefrWO7A9euXat5xYoVXrlp06ZpfuSRRzTHV+K23Z/ff/+95nnz5nnl7IrdM2bMSKvuSF8mqy5/+eWX3rG9f8iMXWk5voFnrVq1NH/11VfeuXRnIn/77bea7eaj2267rVfObg790ksvpXXtioI3PQAAIAg0egAAQBBo9AAAgCAU9Zgea8qUKZpPO+0079xxxx2n2U5tP//8871yu+22m+Y2bdpku4oFzY6psNMqRUQWLlyo+Z///Gfe6mTFd37v06dPieXiOz5fffXVuapSIuwqrnPnzvXOHXTQQWW+3tdff+0d212V7bidjz76qMzXjjvvvPM02/ELIv87fgTZZXfmji/rkEqqqezInF1hPL50wMsvv6w5vvyEXc7FPqOPPfaYV27x4sWahw8frjk+pseeKzS86QEAAEGg0QMAAIIQTPeWFd+E8Mknn9Q8dOhQzXaVVxGRFi1aaG7VqpXmcePGZbeCRcaunpvPVa1tl9Z1113nnevZs6dmO/05vpzBzz//nKPaJe+OO+5IugplYpeWiEt3GjXSZ5eeKG2DV8t2ncycOTPrdcKf7ArlIv/b5ZsJ+zuuZcuWmuNdmoXcncybHgAAEAQaPQAAIAjBdG/ZlWNPOeUU79z++++vOd6lZdnZKO+8804Wa1fc8rkKs30lb7uwTj/9dK+cfQ1/8skn575iyKkXXngh6SoUnTFjxmjecsstU5az3Sx2k2cUHjsLt7SV6pm9BQAAUMHR6AEAAEGg0QMAAIJQVGN66tev7x1fcsklmk888UTN22yzTVrX++2337xjO9063VVJQ2F317ZZxF85tHv37ln93ssvv9w7tlPTq1evrvnpp5/2yp155plZrQdQbGrUqKG5tJ93gwcP1lzMSzyE4PXXX0+6CjnHmx4AABAEGj0AACAIBdm9ZbunzjjjDM0XX3yxV65u3bplvvaECRM033rrrd65fE69LjR2SmN8eqO9X/fcc4937pFHHtH8448/am7WrJlXrnPnzpqbNGmieYcddvDK2Q0w7ava++67r/R/ABSUeBeq3Qz4ww8/zHd1ioLdbFlEZIMN0vtv4g8++CAX1UECjjzyyKSrkHO86QEAAEGg0QMAAIJQYbu3ateurblRo0beuUGDBmlu0KBBma8d36jtzjvv1GxX6mWGVnZUqlRJ80UXXeSds6shL1++XLPtrihNvCtj7Nixmnv37l2meqJwxLtQ0+2Kgc+uYN6mTRvvnP35t3r1as12tpaIyPfff5+j2iHfdt1116SrkHP8pAAAAEGg0QMAAIJAowcAAAQh0TE9W221leYHHnjAO2f7mnfZZZeMrm+nUg4YMEBzfNXJlStXZnR9/MmOrfn444+9c3YX+zg7nd2O44qz09ntDr/ZXuEZhal58+aaH3vsseQqUmC22GILzaU9f/Pnz9fco0ePnNYJyXn33Xc123FyxTS+lTc9AAAgCDR6AABAEHLevXXggQd6xz179tR8wAEHaN5+++0zur7tmho4cKB3rm/fvpp/+eWXjK6P9MybN0/zSSed5J07//zzNdsNQUsTv5dDhgzRPGvWrEyqiCISX5EZQPlNmTJFs/05Gx9iYqe2L1q0KPcVyyLe9AAAgCDQ6AEAAEGg0QMAAIKQ8zE9J554YqnHqUyfPl3zSy+95J377bffNPfv31/z0qVLM6kismzBggXecZ8+fUrMQFn861//0nzqqacmWJPiMWPGDM3x3dIPOeSQfFcHFYgdEzt06FDv3K233qr5kksu0Txt2rTcV6yceNMDAACCQKMHAAAEwcV3Ky61sHPpF0bWRFGU9fm53MvETIyiaL9sX5T7mQyezaKS9WezkO9ltWrVNI8YMcI7d/jhh2seNWqU5q5du3rlElwqJuW95E0PAAAIAo0eAAAQBLq3CgCv0IsK3VtFhGezqNC9lYLt6hLxZ29deOGFmvfaay+vXIKzuejeAgAAYaPRAwAAgkCjBwAABIExPQWAcQNFhTE9RYRns6gwpqd4MKYHAACEjUYPAAAIQlk3HP1BRObmoiJIqU6Orsu9TAb3s3hwL4tLLu4n9zIZKe9lmcb0AAAAFCq6twAAQBBo9AAAgCAUfaPHOTfHOTfFOTfJOTch6fqgfJxzRznnZjrnvnDOXZ10fVA+zrlKzrlPnXMvJ10XZM4594hzbqFzbmrSdUH5Oee6O+emOuc+c85dmnR9sqnoGz3r/V8URU1zsT4K8sc5V0lEBovI0SLSUEQ6OOcaJlsrlFN3EZmedCVQbo+JyFFJVwLl55xrLCLnisgBItJERI51ztVLtlbZE0qjB8XhABH5IoqiL6MoWi0iw0Xk+ITrhAw553YQkWNEZGjSdUH5RFH0jogsTroeyIo9RGR8FEUroihaKyJvi8hJCdcpa0Jo9EQiMsY5N9E5d17SlUG5bC8i35jjeev/DIXpbhG5UkTWJV0RAGqqiBzqnKvhnKsiIm1FZMeE65Q1ZV2npxAdEkXRfOfc1iLyhnNuxvr/KgGQEOfcsSKyMIqiic65VknXB8Dvoiia7py7Q0TGiMgvIjJJRH5LtlbZU/RveqIomr/+/xeKyPPyexcJCtN88f+LY4f1f4bCc7CItHPOzZHfuykPc849lWyVAIiIRFH0cBRF+0ZR1EJElojI50nXKVuKutHjnNvMOVf1jywiR8jvr+5QmD4Wkd2cczs75yqLSHsReTHhOiEDURT1iqJohyiK6srv93FsFEWdEq4WABFZ3zMizrmd5PfxPM8kW6PsKfburdoi8rxzTuT3f9Znoih6LdkqIVNRFK11zv1dRF4XkUoi8kgURZ8lXC0geM65YSLSSkRqOufmicgNURQ9nGytUA4jnXM1RGSNiFwcRdHSpCuULWxDAQAAglDU3VsAAAB/oNEDAACCQKMHAAAEgUYPAAAIAo0eAAAQBBo9AAAgCGVap8c5x/z2BERR5LJ9Te5lYn6IoqhWti/K/UwGz2ZRyfqzyb1MTMp7yZseIL/mJl0BACXi2SweKe8ljR4AABAEGj0AACAINHoAAEAQaPQAAIAg0OgBAABBoNEDAACCQKMHAAAEgUYPAAAIAo0eAAAQBBo9AAAgCGXaewvIl4EDB3rH3bp10zx16lTNxx57rFdu7lxWkgeAQvLmm29qds7fzu6www7L6nfxpgcAAASBRg8AAAgCjR4AABCEIMf0VK1a1TvefPPNNR9zzDGat956a6/cgAEDNK9atSpHtQtX3bp1NXfq1Mk7t27dOs177LGH5gYNGnjlGNNTcey+++6aN9poI+9cixYtNN93332a7X3O1OjRozW3b9/eO7d69epyXz908Xt50EEHae7bt6/mgw8+OG91QmG56667vGP7d+iJJ57I6XfzpgcAAASBRg8AAAhCUXdv7bzzzpqvvPJKzc2bN/fKNW7cOK3rbbPNNprtFGpkx6JFizS/88473rl27drluzpIQ6NGjbzjLl26aD711FM1b7CB/99X2223nWbbpRVFUbnrZP+uDBkyxDt36aWXal6+fHm5vytE1atX947feustzd99951m+/Myfg7huf322zVfcMEF3rk1a9ZottPXc4E3PQAAIAg0egAAQBAKvnvLzt6xr65F/BlAm2yyieb4io/ffPON5p9++kmznSUkInLaaadptjNOZsyYUdZqowS//PKLZmZhFYbbbrvNO27btm1CNSnZmWee6R0//PDDmt9///18V6fo2S4turdgNWvWTHN8BuB7772necSIETmtB296AABAEGj0AACAINDoAQAAQSiIMT3xKZJ33HGH5tNPP11zfKXlVGbNmuUdH3nkkZorV66sefr06V65mjVrlpiRHVtssYXmJk2aJFgTpOuNN97wjlON6Vm4cKF3/Mgjj2i2Y+xKm7Jul5po2bJlmeqJ/IiPl0TFZldGFxG59tprNXfo0EHz4sWLM7q+vYZdGmb27NleuR49emR0/UzwpgcAAASBRg8AAAhCQXRvnXjiid7xOeecU+Zr2Ndpbdq08c7ZKeu77bZbma+N7KhSpYrmnXbaKa3P7L///t6xXT6Aae+5d//993vHL7zwQonl7IqrIplNX65WrZrmqVOneufsCs+l1WfChAll/l6kz3ZPbrrppgnWBOl48MEHvWP7+69hw4aa7ZTysrDdZTVq1NB87rnneuUmT56c0fUzwZseAAAQBBo9AAAgCDR6AABAEApiTI/drbk0c+bM8Y4//vhjzVdddZVmO4Ynzm5rgfz69ttvNT/22GPeuT59+pT4mfifL126VPO9996braohhbVr13rHpT1b5WWXlthyyy3T+sy8efO841WrVmW1Tkht33339Y4//PDDhGqCVFasWOEd2zFZduumdDVt2tQ7tmMz161bV65rZwtvegAAQBBo9AAAgCAURPdWfHrbeeedp3nMmDGav/jiC69cfBXYdNSuXbvMn0H23Xzzzd5xqu4tFLf27dtrtj8H0p0O3bt376zXKXTxLs1ly5Zptqvn77rrrnmrE9Jnf7buueee3jm75Ee608g322wzzXYYiYi/DMlHH32k+bnnnkuvsjnAmx4AABAEGj0AACAIBdG9ZWf1iOS2q8NuaoiKY4MN/myf21kAKHwdO3bU3KtXL++c7SLZaKON0rrepEmTNMdXgkb52RmSIiLvvvuu5mOPPTbf1UEadtxxR822mzjeVXnxxRdrXrRoUVrX/sc//qE5PtPa/u4++OCD06tsjvGmBwAABIFGDwAACAKNHgAAEISCGNOTqW7dumm20+qcc145uwplfAqf9cEHH2hmddH8suN47P1CsurWresdd+7cWfPhhx+e1jUOOeQQzene2+XLl3vHV199teZXX31V88qVK9O6HlBM4r/HRo0apblmzZqaBw0a5JV7++2307p+jx49NHfp0iVluVtvvTWt6+UTb3oAAEAQaPQAAIAgFGT3ll3lsVGjRprjq6+2bdu2xM/b6c8iqadAL1iwwDvu2rWr5t9++y29ygJFxr46Hz16tHfObjCYS3aatIjIgw8+mJfvRfpq1KiRdBWK2oYb+r++O3XqpPnhhx/2zqVa8iO+RMs111yjecCAAZq32morr5ydmm6HizzxxBNeuQceeCD1P0BCeNMDAACCQKMHAAAEocJ2b9nVV/fee2/v3MiRIzVvu+22muMzNWz3lJ15ddRRR3nlbHeZValSJe/4pJNO0jxw4EDNq1evLvHzQLGLz4SMH6cjk9W24yv/2q5sO3sLyWnXrl3SVShqdjNeEZGhQ4dqjs+CtM+V3Zh7v/3288rZY3v/tt9+e6+c/b1rV24+66yz0qp7knjTAwAAgkCjBwAABIFGDwAACEKFGdNTuXJl79iOu7GrScbdeOONmseOHeude//99zXbKXfxco0bNy7x2rVq1fKOb7vtNs1ff/215hdeeMErt2rVqpT1RWbSHffRokULzffee29O6xSqKVOmaG7VqpV3zk6bff311zX/+uuvGX3X2WefrfmSSy7J6BrIrbfeekszu6zn1umnn6750Ucf9c6tWbNG89KlS71zZ5xxhuYlS5ZottPSRURatmyp2Y7vKW0XA7vC8zfffOOVsz8fZs+eLRUBb3oAAEAQaPQAAIAguLJs3uicy+pOj3Za+k033eSd69mzZ8rPvfbaa5rt6/T4Kz3bPWWnse6zzz5eOTvlvF+/fprj3V7HH398ifX597//7R3ba9hXiXGffvppynNWFEVlnwf8F7J9L3PNroCd7t/ZvfbaS/O0adOyXqcMTYyiaL+/LlY2hXY/01W9enXNP/74Y8pydnptPqes82yKnHzyyZqfffZZzfElRBo2bKh57ty5ua9Y2WX92cz2vbRDM+rUqeOds5t7PvLII2ldz94TEX9l82bNmmkurXvLeuaZZ7zjM888M6165EDKe8mbHgAAEAQaPQAAIAh5n71lVzm++eabNffo0cMr98svv2ju1auXd27YsGGabZfW/vvv75UbNGiQZruq86xZs7xyF154oWY7E6FatWpeuYMOOkhzx44dNcdXHh0zZoykYke377zzzinLwTdkyBDN559/flqfOe+88zRfeumlWa8Tcu/II49Mugr4C2vXri3xz+NdIhtvvHE+qlPU7Aa/8VnN8ZlT6bAzr0T8DbytDh06eMdTp04tsdy8efPKXId8400PAAAIAo0eAAAQBBo9AAAgCHkf02PHWdhxPCtWrPDK2XEb8TEydipd165dNdudlkVENtlkE812Snx8JctUfaHLly/3ju1UeZvj/Z12vE/cZZddlvIcUpsxY0bSVQiKXU7iiCOO8M7ZabPxacnlFd+l+e67787q9ZF9dpyJfU4bNGjglbPj6i666KLcV6wIDRw4sNzXsMtAnHbaad45O47VrqA8YsSIcn9vRcGbHgAAEAQaPQAAIAh5X5F5wYIFmu2KyfFNOu1r0s0228w7V69evbS+q0+fPprtZqF2dd9CwKqvvs8//1zzrrvumrKc3aQ0/ncmwc3vKuyKzIceeqjma665RnObNm28cnaphUymyYr4GwDbbmm7zISISNWqVUv8fLxbzS4bYZedyDWeTZ/tjrRDD0REateurTnTDWhzrMKvyJwNdgkYu2yMiMiiRYs02yVgCmEqegwrMgMAgLDR6AEAAEHI++yt7777TrPt3oqv1tmkSZOU17AbCr7zzjuaX3jhBa/cnDlzNBdalxZS++yzzzTvsssuKcutW7cuH9UpGrZrKb7ZrnXllVdq/umnnzL6LttlZjcALq27fdy4cZrvv/9+71w+u7SQnvi9tBs7I7/s5qTnnHOO5vg9shuOFmCXVlp40wMAAIJAowcAAASBRg8AAAhC3sf0tGjRQvMJJ5yg2fbri4gsXLhQ8yOPPOKdW7JkiWb6icNj+52PO+64BGsSpgsvvDBn17bPvYjISy+9pLl79+6aK+iUZxh2dV8R/+d9fIdw5NYbb7yh2Y7veeqpp7xyN9xwQ97qlBTe9AAAgCDQ6AEAAEHIe/eWneL65JNPlpiB0kybNk3z9OnTvXN77LFHvqtTNOwKun//+981/+1vfyv3teMrYNsNht99913NDz30kFduypQp5f5u5I/dwDK+yr59bpFfjz32mGa7+faLL76YQG2SxZseAAAQBBo9AAAgCDR6AABAEPK+yzrKjp2ci0qF3WXdstvCdOnSxTt3yy23aN5yyy29c3YrGDtNdvTo0V45ux1NIePZ9A0fPlxzfHxdu3btNM+dOzdvdSqDIHZZDwS7rAMAgLDR6AEAAEGge6sA8Aq9qBRE9xbSw7NZVOjeKh50bwEAgLDR6AEAAEGg0QMAAIJAowcAAASBRg8AAAgCjR4AABAEGj0AACAINHoAAEAQaPQAAIAgbFjG8j+ISIXcKa6I1cnRdbmXyeB+Fg/uZXHJxf3kXiYj5b0s0zYUAAAAhYruLQAAEAQaPQAAIAhF3ehxzu3onHvLOTfNOfeZc6570nVC5pxzjzjnFjrnpiZdF5SPc24T59x/nHOT1z+bNyZdJ2SOZ7P4OOcqOec+dc69nHRdsqmoGz0islZEroiiqKGINBORi51zDROuEzL3mIgclXQlkBWrROSwKIqaiEhTETnKOdcs4Tohc48Jz2ax6S4i05OuRLYVdaMniqIFURR9sj7/JL/fwO2TrRUyFUXROyKyOOl6oPyi3/28/nCj9f9jVkWB4tksLs65HUTkGBEZmnRdsq2oGz2Wc66uiOwtIuOTrQkAEX19PklEForIG1EU8WwCFcPdInKliKxLuiLZFkSjxzm3uYiMFJFLoyhannR9AIhEUfRbFEVNRWQHETnAOdc46ToBoXPOHSsiC6Momph0XXKh6Bs9zrmN5PcGz9NRFI1Kun7527YAACAASURBVD4AfFEULRWRt4QxIUBFcLCItHPOzRGR4SJymHPuqWSrlD1F3ehxzjkReVhEpkdR9I+k6wPgd865Ws65LdbnTUWkjYjMSLZWAKIo6hVF0Q5RFNUVkfYiMjaKok4JVytrirrRI7+3WDvL7y3VSev/1zbpSiEzzrlhIvKhiNR3zs1zzp2ddJ2QsW1F5C3n3H9F5GP5fUxPUU2NDQnPJgoF21AAAIAgFPubHgAAABGh0QMAAAJBowcAAASBRg8AAAgCjR4AABAEGj0AACAIG5alsHOO+e0JiKLIZfua3MvE/BBFUa1sX5T7mQyezaKS9WeTe5mYlPeSNz1Afs1NugIASsSzWTxS3ksaPQAAIAg0egAAQBBo9AAAgCDQ6AEAAEGg0QMAAIJQpinrAABkYvfdd9f82muvaa5UqZJXrk6dOnmrE8LDmx4AABAEGj0AACAIdG8BALJu0KBB3vHpp5+ueautttL88ssv561OAG96AABAEGj0AACAIBR891bDhg01H3vssd65c889V/PHH3+sedKkSSmvd/fdd2tevXp1NqoIAEWrdu3amkeNGqW5WbNmXrko+nPvzalTp2o+++yzc1g7wMebHgAAEAQaPQAAIAg0egAAQBAKckzP+eefr/nOO+/UvPnmm6f8zK677qq5ffv2KctNmDBB89ixYzOtIlAh2WfETiEWEfn1118177vvvpqrVq3qlevYsaPmcePGeefmz59f5jp99913mkePHu2ds88jKga7srKISP/+/TUfeOCBKT/Xq1cvzfa+/vjjj1msHf6Kc07zsGHDvHNt27bVbMfLzps3L/cVyxPe9AAAgCDQ6AEAAEFwdhrhXxZ2Lv3COWRX85w2bZrmrbfeutzXXrp0qeZ4N9iYMWPKff1MRFHk/rpU2VSUexmgiVEU7Zfti6Z7P/v166e5R48e2a5Gua1bt847ts/38OHDNcdfy3/11Ve5rVgKIT6bzZs3947ffffdEsvZbhQRkU6dOmmO378KIuvPZkW8l1WqVNH8+eefe+e22247zeedd57moUOH5r5i2ZXyXvKmBwAABIFGDwAACEJBzt5avHix5j59+mi2swhE/Nd4X3/9teaddtop5bW32GILzUceeaR3LqnuLeRWnTp1NG+66abeuQ4dOmi+8MILU17jlVde0dy1a9cs1i67TjrppDJ/Jj675r///W+ZrzFz5kzvuH79+prtM7f33nt75Ro3bqz5lltu0Tx58mSvXFLdW6GwM7aefvpp71y8G+sP8b9r8Zl5SMaKFSs0l9a9lY3hIhURb3oAAEAQaPQAAIAg0OgBAABBKMgxPdaQIUM025WaRUSaNGmiefny5WW+9uDBgzOvGCqUww8/3Du24w3suJ3q1at75dJd0iG+o3RFZcep2XE1Iv877uYPdgyAiMiCBQuyWie74vOUKVO8c6nG37Vr1847tmOqkH2dO3fWHL8nr776quYLLrhAcyarcyO/4r/jWrVqpblBgwZ5rk1+8KYHAAAEgUYPAAAIQkGuyJzKKaec4h1fc801mps2bVrm6zVq1Mg7nj59emYVK6cQV33NlF05dM8999S8//77p/X5n376yTu203Pjm18+88wzmu1mnX8h0RWZK6IzzjhD81NPPZWy3KpVqzS3aNHCO/fxxx9nv2JpKOZn84MPPtBsf35+++23Xrmjjz5a86xZs3JfsdwJYkVma8cdd/SO586dq3n16tWad955Z69ctru4c4AVmQEAQNho9AAAgCDQ6AEAAEEo+Cnr1nPPPecdv/fee5pff/11zXasR2luuukm7/jUU08tR+2QLTVq1NB82223eefOOusszXa7kokTJ3rlbr/9ds1Tp07VvHLlSq+c3b4EmatcubJ3fM8992g+88wz07rGQQcdpPnTTz/NTsWgjj/+eO/4wAMP1GzHfj777LNeufgzg8JltxSxz2x8iYgHHnggb3XKNt70AACAINDoAQAAQSiq7q2OHTt6x3vttZdmu1tzut5///1y1wnZd/3112s+++yzvXODBg3SfO2112r++eefc18xeA477DDNnTp18s516dKlxM+sWbPGO+7WrZvmpJaMKGZ2h/tDDz00rc8sWbLEO543b16Zv7d79+6a49OmrR49epT52shcqiVs4t3ThYw3PQAAIAg0egAAQBAKsnvLboQ2atQozfXq1fPKbbhh+f7xXnzxxXJ9HmVTpUoVzVdddZV3zm54eOmll2p+6623vHJ2ll4ZVklGlhxwwAGa7b2oVKlSWp+Pv17/5ptvNP/222/lrB3i7L/Tfffd1zu3wQZ//jfxunXrNL/zzjtpXfvyyy/3ju29veSSSzTXqVMn5TWuuOIKzTvssIN3jg1NkQne9AAAgCDQ6AEAAEGg0QMAAIJQkGN69thjD81299fyjuGJs2NHRPzps8i+6667TnN8TM+IESM0jxkzRjPjdiqW0047TXO643is+NTYl19+WbPd5f6ll17yyr3wwguap0yZUubvDVXLli01x6es23E8dmXyH3/8MeX17G7shxxyiHcuvqrvH3755Rfv2E6Br1+/vub4ivvt27fXbHcHB0rDmx4AABAEGj0AACAIBdm99fzzz2u23SB2E0kRkU022aRc37PtttuW6/Mom169emmOT10eNmyYZrq0Ki67hITtht5///29cjVr1izztffbb78Ss4jIDTfcoPnuu+/W3K9fP6/cwoULy/y9xaRq1aresR0eELdgwQLNTz75pOZZs2Z55XbffXfNPXv21BzfwPSHH37Q/MYbb2geMGCAV65atWqax44dq7l69eop64rssBuOplqdudDxpgcAAASBRg8AAAhCQXZvWffcc4/m+GtXu5meFZ/lZTeptK9WkV//+c9/NMe7L+69917NK1eu1GxfkyN5H3zwgeZjjjlG80477eSVs91btWvX1nzSSSd55c466yzN9tV7nF092K4EHF9luHXr1prt7KRQxGdU3XXXXSnLPvjgg5pvuukmzfZ+iYj0799fc9u2bTX/9NNPXrlnn31Ws11pebfddvPKDRkypMRr2K4uEWZs5UKxdmlZvOkBAABBoNEDAACCQKMHAAAEoeDH9Fj/+te/0ioXHxuw6667au7du7dmu7qoiL8bMP3J6TvwwAM1f/rpp9651atXaz766KM1x1e/vv766zXblVmbNWvmlZs+fXr5KoucsCv6lnT8h/gzPG7cOM12Z267m3tp7IrDIiI9evTQHJ/OHoK99tor7bJ2HI9llyUQ8Z9vKz5l/e2339bcvHlzze+++27KOtjlB+y9Q37997//TboKWcObHgAAEAQaPQAAIAhF1b2VrvimhrZLy1qzZo13/Ntvv+WsToXOrl5tN4kU8acrX3bZZd65p556SvPixYs12ynqIn731uabb655yy23zLDGKARPP/205n/+85+a//3vf3vlWrRokdb16tWrl52KFaj4Mh62q3/06NEpP2e7+uvWrZvyGnYquu3OEvFXbrb3NT7cwF7Ddm8hObNnz066ClnDmx4AABAEGj0AACAIQXZv3XzzzWmVe+SRR7zjefPm5aI6ReGTTz7RHF/V2m4Ka7uzSnPppZemPGe7NqZOnZpuFVHg1q5dq3nixIneuXS7tz7//POs1qnQ2RV4012NN76Stf2cnR0Wn6FnN4D+6quvNB966KFeuWXLlqVVDyATvOkBAABBoNEDAACCQKMHAAAEIdExPTVq1NAcHz9jp6c+88wz5f4uO6X6vPPOS+sz8ZVHkZrd7f66665Lec7muFmzZmmO77xsV8Du1auX5uXLl5e9sigX+yyde+653rkZM2ZoHjFiRFa/t1KlSpqbNGmS1mfsOCARkfHjx2e1ToXmxRdf9I579uypOb6Csl012f77rlq1asrrn3nmmZrjU9F/+OEHzTfeeKPm+fPn/1W1kbCNN9446SpkDW96AABAEGj0AACAICTavTVw4EDNxx13nHfOrt4Zf/1pj7/44gvN++67b8pr2Ne48SnV1oABAzR/++23KcvBd9ttt2mOr2S99957az788MNTXsOurvzqq6965+wqrfaeI/e22WYb7/i1117TvOeee3rnsr1Cdu3atTVffvnlmg877LC0Ph/fgLa0zS1DYDf4FRFZsWKF5ipVqnjn3nvvPc3pTme3fvrpJ+/42Wef1Rx/vlGxtW3b1jseNGhQQjUpP970AACAINDoAQAAQUi0e2vw4MGad955Z++cnTnw1ltveefmzJmjedq0aZrjK3ummmUQf1VrZ5z06dNH86+//pqi5ihN//79k64Csii+6WO8S8uyz/HMmTM1r1y5MuVnNt10U81XXnmld852aZU2a8jOFLLdKt26dUv5mRDFV7Lu0KGDZvvvWkSkVatWaV3z8ccf1zxlyhTNn376qVcuvgEpkvf99997x/b3acOGDfNdnbzgTQ8AAAgCjR4AABAEGj0AACAIrixTEZ1zZZ+3mKb4OJDZs2drtmN/smHx4sXecc2aNbN6/WyLosj9damyyeW9RKkmRlG0X7Yvmsv7GV91+YEHHkjrc3ZMR2k7Z1evXl2zXd6gLH7++WfNJ554ouY333wzo+uli2ezqGT92Sy0e/nxxx9rtkvAvPzyy165du3a5a1OGUp5L3nTAwAAgkCjBwAABCHRKetWjx49vGO7wdnmm2+e8nNNmzbVbKdfxtnX60cccUQmVQSC9O9//9s7Hj58uOb27dun/FymXVWp2M1D49PoR44cqTn0TUWBTE2aNEmz7d4q7XdwoeFNDwAACAKNHgAAEAQaPQAAIAgVZso6UmNabFEpuCnrcXa8nZ0eLuLvfv75559rLm2Kq90GJm7s2LGa7bYW8S0OksKzWVSCn7Jet25dzcOGDdNstxoRERkyZEi+qpQppqwDAICw0egBAABBoHurAPAKvagUfPcW/sSzWVSC794qInRvAQCAsNHoAQAAQaDRAwAAgkCjBwAABIFGDwAACAKNHgAAEAQaPQAAIAg0egAAQBBo9AAAgCBsWMbyP4jI3FxUBCnVydF1uZfJ4H4WD+5lccnF/eReJiPlvSzTNhQAAACFiu4tAAAQBBo9AAAgCEXd6HHObeKc+49zbrJz7jPn3I1J1wnl45yb45yb4pyb5JybkHR9kBmezeLinNvCOfecc26Gc266c6550nVCZpxzjzjnFjrnpiZdl1wo6jE9zjknIptFUfSzc24jEXlPRLpHUfRRwlVDhpxzc0RkvyiKfki6Lsgcz2Zxcc49LiLvRlE01DlXWUSqRFG0NOl6oeyccy1E5GcReSKKosZJ1yfbyjp7q6BEv7fofl5/uNH6/xVvKw8oEDybxcM5V11EWohIFxGRKIpWi8jqJOuEzEVR9I5zrm7S9ciVou7eEhFxzlVyzk0SkYUi8kYUReOTrhPKJRKRMc65ic6585KuDDLHs1k0dhaRRSLyqHPuU+fcUOfcZklXCihJ0Td6oij6LYqipiKyg4gc4Jwrutd1gTkkiqJ9RORoEbl4/atYFCCezaKxoYjsIyL3R1G0t4j8IiJXJ1sloGRF3+j5w/r+5bdE5Kik64LMRVE0f/3/LxSR50XkgGRrhPLi2Sx480RknnlT95z83ggCKpyibvQ452o557ZYnzcVkTYiMiPZWiFTzrnNnHNV/8gicoSIFOUMg2LHs1k8oij6TkS+cc7VX/9HrUVkWoJVAlIq6oHMIrKtiDzunKskvzfwRkRR9HLCdULmaovI879P/JENReSZKIpeS7ZKyBDPZnG5RESeXj9z60sR6ZpwfZAh59wwEWklIjWdc/NE5IYoih5OtlbZU9RT1gEAAP5Q1N1bAAAAf6DRAwAAgkCjBwAABIFGDwAACAKNHgAAEAQaPQAAIAhlWqfHOcf89gREUeSyfU3uZWJ+iKKoVrYvyv1MBs9mUcn6s8m9TEzKe8mbHiC/5iZdAQAl4tksHinvJY0eAAAQBBo9AAAgCDR6AABAEGj0AACAINDoAQAAQaDRAwAAgkCjBwAABKFMixMCAJCJXXbZRfNtt92m+cQTT/TK7bXXXppnzJiR+4ohKLzpAQAAQaDRAwAAgkD3FgAg6w466CDv+LXXXtO8aNEizYMHD/bKff/997mtGILGmx4AABAEGj0AACAINHoAAEAQGNODCqNz586ajzzySO9ckyZNNNevXz/lNT766CPNxx13nOZly5Zlo4qooDbbbDPN48aN07zddtt55Q4++GDNc+bMyXW1gnPMMcdofu6557xzQ4YM0XzttddqXrFiRe4rBqzHmx4AABAEGj0AACAIdG8hr2rWrOkdDx06VLPtjlq6dKlX7sMPP9Q8d+5czS1btvTKHXLIISV+pmHDhhnWGPkU746qVatWieWWLFniHf/f//2f5n333VfzzJkzvXI//vhjeauImN12203ziBEjNL/99tteuSuuuELzunXrcl8xoAS86QEAAEGg0QMAAIIQZPeWfc0qIlK5cmXNe+yxh+aOHTumvIbdCK9Ro0ZZrF1xs6uyiojUrVtXc79+/TTfeeedXrnFixeXeL0GDRp4x//5z38077777pp79+7tlbvpppvSqzAytueee2q+5JJLvHN16tQp8TP2nomI7LTTTiWWu/32271j233pnNM8f/58r5x91pGZTTbZxDt+6KGHNE+ZMkXzaaed5pWjS6vi22qrrTSffvrpmq+55hqvXLwb+g/XX3+9d9y3b98s1i47eNMDAACCQKMHAAAEgUYPAAAIgouiKP3CzqVfOAHx6cuNGzcu8dyJJ57olbNjANJl+6e/+OIL71y2p0dHUVT2Cv6FfN7LNm3aaI6P6bFTXDt06FDu77Jjda677jrNdpq7iMjOO+9c7u/K0MQoivbL9kUr4rPZrVs3zXfddVdan1m1apV3/Oyzz2pu3bq15m233TblNezzfOaZZ3rnnnrqqbTqka5CfzYzER9v9/e//12znb4+b968vNUpS7L+bFb0e9m8eXPv+B//+IfmAw44QHNZ2gnWk08+qblr164ZXSNDKe8lb3oAAEAQaPQAAIAgVNgp6/b19bBhw7xzu+yyS4mfqV69undsNyG0r7wnTpzoldtnn33KXL8NNvizvWi/B/9ro4020hzvChw+fHhWv8tucmi7t+LTbKtVq6Z5+fLlWa1DyPr06aO5Z8+eKcs9/vjjmhctWqS5f//+Xjl7rmnTpppff/11r5xd6dt+Jr7pJTKz8cYba+7UqZN3zm7wWoBdWsGxz8qDDz7onbNLttjn6IUXXvDKjR49WrPtQj711FO9cs2aNdNsl4tYvXp1WaudNbzpAQAAQaDRAwAAgkCjBwAABKHCjOk5/PDDvWO7tPmOO+5Y7uvbaeQ//PCDd872cdrltR999FGv3A477FDitadNm1bu+hWzsWPHat577729cytWrMjqd8WnPP+hdu3a3vEZZ5yheciQIVmtQ8js+LZNN91Uc3zJgGuvvVbzggULUl6vXr16mu1S+PHd1+3foxtvvFHzr7/+mk618ReuvPJKzZtvvrl3zt5LVHx2PI4dwyMiMmbMGM1t27ZN63p2nGb897j9nWm/a/LkyelVNgd40wMAAIJAowcAAAShwnRv2denIul3adnujKuuuso7N378eM0zZ85MeY0ff/xRc/fu3TWn6s4SEZkzZ47mzp07p1XXUOWzi+HLL7/UbLsd46tk25VjkT12ivjRRx+tOf4a3e6SftFFF2mOLzthV4g95phjNC9evNgrd+utt2q+7777ylpt/IUjjjhC8/vvv++d++STT/JdHZTDypUrU56zXV/ZYJcDiQ8rSQpvegAAQBBo9AAAgCAk2r1lX5nalRv/ytdff63Zdi3FX7tmorQuLcu+Bqwor+0gsmbNmhIz8mPSpEmaP/zwQ83x7i27eajdkDa+MelOO+1U4vfYGVoiIoMGDSp7ZVGqQw89VLP9+bzXXntldL1WrVpptqv9fvbZZxldD5mxuxPEN9tesmSJZruK/a677uqV69Kli+Z9991X83fffeeVs7Nk58+fn1mFs4w3PQAAIAg0egAAQBBo9AAAgCAkOqbniiuu0FylSpWU5T744APv2PbnZzKOZ8stt/SO7dTaFi1apFWPV199tczfi9yzu0HHd1a3fvrpp3xUJzh2CYnSdq/fdtttNY8cOVJzfIxBFEWaH374Yc3xXZ+RfR07dtQ8ffp0zXZZiDg71mPAgAHeOftz1/496dGjh1du8ODBZa4r0teoUSPN9vkSEbn88ss129/PdtxOXPv27TXbJSsqKt70AACAINDoAQAAQUi0e+vBBx/UbDf9FBFZtmyZZjvtTeR/p8WV1QUXXOAd33zzzSWWi0+lPO2007JWB+RG3bp1NdevXz9luddeey2t69m/l02aNPHONW/eXPOzzz6rubTVv0MS32Q0E7YbuX///pq/+eabcl8bpTvrrLM025/B8U19K1eurPmGG27QfP7553vlXn/9dc12M8v4xs6zZ8/WnO5zivTZHQiqVq3qndtvv/00267meDeY3eC30Dbc5k0PAAAIAo0eAAAQhES7t+ysDZtz4bjjjtPcu3fvlOXWrl2r+YEHHvDO0aVVMdgZWvEVtA8++OC0rjFkyBDNEydO1LzPPvt45bbaaivN8U1w7QywevXqabYzWEJTqVIlzXZF3/isrFReeeUV79g+t8gtO6tHRGTDDf/89WB/LsbZZ8Z2R5U2k+ef//yn5kMOOcQ716tXrxKvh+yw9zm+E4L9eWrvUdyoUaM0070FAABQAdHoAQAAQaDRAwAAgpDomJ58siu4xqffWd26ddNsp9Qjc5tuuqnmrbfe2jtnV/o88MADNR922GFpXa9hw4YZ1cn2a1evXj1luUceeURzfLyJnfr51VdfZVSPYjN8+HDNJ510kubSnjkr3XLIvm222SbludKWYbBLe1x33XVl/t7777/fO54yZUqZr4HMfPTRR97xnnvumdbn+vbtm4vq5AVvegAAQBBo9AAAgCAUdfeWfQW3wQZ/tu/WrVuX8jNvv/12TutUrGyXU58+fbxzdtpxgwYNMrq+3bzy559/1hyfSmun2VpDhw71ju2U9U8++SSjOoVqu+2209y1a1fv3Mknn6zZdlXF/x1Pnjy5xGvEuz9RMcybNy/lufJu3lvatZFfjRs31pzu78xCw5seAAAQBBo9AAAgCEXVvWU3vhMR2XvvvTXb13PxGSLdu3fXPGvWrBzVrrjZ2XFt2rTxztkNCuMzoOysp9GjR5f4GRGROXPmaLavw2fMmOGV23333TV/+eWXmi+//HKvnO0iQ9m0bt1a80033ZSynJ3Jc++993rnTjjhBM22e6vQVnctJvFVs9NdRbu8WrZs6R2Xt7sMmVu5cqVm+ztz3LhxXrnVq1fnq0pZx5seAAAQBBo9AAAgCDR6AABAEAp+TE+VKlU0d+rUyTsXH1vyh2HDhnnHTz/9tOZimpqXT0cccYTm+OrEdhrzp59+mtH17VT0O+64Q3N8l/WFCxdqPu200zQzhidzrVq18o7vueeelGXbtWun+d///rfm+Gq/vXv3LvHzduwW8is+1jGXq2NvtNFGmi+44ALv3JNPPpmz74Vvjz328I7PPvtszYsWLdIcXzW7kJ9T3vQAAIAg0OgBAABBKMjurapVq2p+6KGHNJ9yyikpP3PZZZdpjk+fpUur/Oyr8KVLl3rnMtlAcJNNNvGOn332Wc3HHHOM5vjU9vbt22tmpeXsiHcT2w1a4yuYv/zyy5ptF8axxx6b8hp2avQPP/xQvsoiY/HlAhYsWKDZDh2Id3Wky/59sNeoW7euV+5vf/tbRtdHeuyz99prr3nntt9+e81XXXWV5ueeey73FcsT3vQAAIAg0OgBAABBKMjuLTtjp7QurdmzZ2subcYJyu/zzz/X3LRpU+/cgw8+qLlGjRreObvxpF1BuWfPnl65+vXrax4/frzmiy66yCuX6ewwpFbarJ74OduFYVddHjhwoFduyZIlmu1msPfdd1/5KouM2e4sEX/D5gEDBqT8nJ39uuuuu2rea6+9vHLXXHON5l9//VWznfkpQhdnrvXr10+z7c4SERk+fLjm0u55IeNNDwAACAKNHgAAEAQaPQAAIAgFMaanQYMG3nF8x+w/2HElIiJHH310zuoEn71HN998s3euR48emjfYwG9nH3XUUSVe78UXX/SOr7jiCs3xaZbIrVq1aqU8Z1dtFRF54403NB966KEpP2d3Vn/ppZfKUTvkyuDBg0v88/hYj/gSIH+I75Zux1Xecsstmgt5x+5Ccfjhh2u2yw/YXdVF/KVBihVvegAAQBBo9AAAgCC4smwq55zL3Q50pbBTIkVETj/99BLLdevWzTsulumvURS5vy5VNkndS8jEKIr2y/ZFc3k/L730Uu+4tKmsdnXlxYsXa453ldx+++2a46/YCwnPZlHJ+rOZ1L2Mr3I9ceJEzXa1+86dO3vlRo0aldN65VHKe8mbHgAAEAQaPQAAIAg0egAAQBAq7JT1Ro0aaa5WrVrKcnaLgzfffDOndQJC9Pjjj3vHlStX1nz99dd75yZMmKDZLjtw11135ah2AERENt10U812mRARf2f1kSNHai6iMTxp400PAAAIAo0eAAAQhAo7Zf2OO+7QbFfjFRGZO3eu5rZt22qeOXNm7iuWAKbFFpWCm7KO1Hg2i0pBT1m/6KKLNA8aNMg79+GHH2pu3bq15lWrVuW+YslgyjoAAAgbjR4AABCECjt7a8yYMZrj3Vt2w9Fi7dICACCVAw44wDu+5pprNNsNXUVEHnroIc1F3KWVFt70AACAINDoAQAAQaDRAwAAglBhp6zjT0yLLSpMWS8iPJtFpaCnrMPDlHUAABA2Gj0AACAIZZ2y/oOIzP3LUsimOjm6LvcyGdzP4sG9LC65uJ/cy2SkvJdlIT0huwAAE4VJREFUGtMDAABQqOjeAgAAQaDRAwAAglDUjR7nXH3n3CTzv+XOuUuTrhcy45zb0Tn3lnNumnPuM+dc96TrhMw55y5bfx+nOueGOec2SbpOyIxzbgvn3HPOuRnOuenOueZJ1wmZc851X/9cflZsvzODGdPjnKskIvNF5MAoihhYVoCcc9uKyLZRFH3inKsqIhNF5IQoiqYlXDWUkXNuexF5T0QaRlG00jk3QkRejaLosWRrhkw45x4XkXejKBrqnKssIlWiKFqadL1Qds65xiIyXEQOEJHVIvKaiFwQRdEXiVYsS4r6TU9MaxGZTYOncEVRtCCKok/W559EZLqIbJ9srVAOG4rIps65DUWkioh8m3B9kAHnXHURaSEiD4uIRFG0mgZPQdtDRMZHUbQiiqK1IvK2iJyUcJ2yJqRGT3sRGZZ0JZAdzrm6IrK3iIxPtibIRBRF80Wkv4h8LSILRGRZFEVjkq0VMrSziCwSkUedc58654Y65zZLulLI2FQROdQ5V8M5V0VE2orIjgnXKWuCaPSsf93aTkSeTbouKD/n3OYiMlJELo2iaHnS9UHZOee2FJHj5fdfmNuJyGbOuU7J1goZ2lBE9hGR+6Mo2ltEfhGRq5OtEjIVRdF0EblDRMbI711bk0Tkt0QrlUVBNHpE5GgR+SSKou+TrgjKxzm3kfze4Hk6iqJRSdcHGTtcRL6KomhRFEVrRGSUiByUcJ2QmXkiMi+Koj/euj4nvzeCUKCiKHo4iqJ9oyhqISJLROTzpOuULaE0ejoIXVsFzznn5PdxA9OjKPpH0vVBuXwtIs2cc1XW39fW8vsYLRSYKIq+E5FvnHP11/9RaxFhckEBc85tvf7/d5Lfx/M8k2yNsqfoZ2+t71v+WkR2iaJoWdL1Qeacc4eIyLsiMkVE1q3/42uiKHo1uVohU865G0XkdBFZKyKfisg5URStSrZWyIRzrqmIDBWRyiLypYh0jaJoSbK1Qqacc++KSA0RWSMil0dR9GbCVcqaom/0AAAAiITTvQUAAAJHowcAAASBRg8AAAgCjR4AABAEGj0AACAINHoAAEAQNixLYecc89sTEEWRy/Y1uZeJ+SGKolrZvij3Mxk8m0Ul688m9zIxKe8lb3qA/JqbdAUAlIhns3ikvJc0egAAQBBo9AAAgCDQ6AEAAEGg0QMAAIJAowcAAASBRg8AAAgCjR4AABAEGj0AACAIZVqRGQAAhOWZZ57xjps1a6a5Q4cOmsePH5+3OmWKNz0AACAINHoAAEAQ6N6K2X333TUPGTLEO9exY0fNCxYsyFudkJlWrVppfvPNN71zG2ywQYnl3n777VxXCwAKSp06dbzjunXran7yySc1N2rUyCu3Zs2anNYrE7zpAQAAQaDRAwAAgkCjBwAABCEnY3qqVq2qefPNN/fOLVu2TPOKFSty8fXl0rZtW80tWrTwzp1zzjmab7vtNs1r167NfcWQli5dumi+5JJLNK9bty7lZ/7xj39ofuKJJ7xzgwcP1sx9BrKvV69e3vGtt96quV+/fpqvvvrqvNUJIjvuuKPm/fbbL2W5evXqad5wQ79JwZgeAACAhNDoAQAAQXBRFKVf2Lm0Ct9yyy2a468ke/bsqfmuu+5K+7vz5dBDD9X81ltvpSzXoEEDzV988UVO6xRFkcv2NdO9lxWd7c4SEencubPmePekZaesl9b1ZV/dzp07N4Ma/o+JURSlflecoWK5n/GpsZdddpnmiy66SHP8Nfrw4cM1n3HGGTmq3f/i2cyMHQIxc+ZM71zt2rU12+6Riy++2Cv38MMPZ7taWX82C/le7rnnnponT56cstwLL7yg+ZRTTvHOlfazNcdS3kve9AAAgCDQ6AEAAEHI+4rMN9xwg+Yvv/xS8+jRo/NdlRLZV6tIzhZbbOEdN23aVPOjjz6quVatWl65jTfeuMTrzZgxwzu23Vt2FW7k31lnnaU53uU9a9Yszeeff75mO7NExP+5ctNNN2mO33ckx3ZJXnjhhZpL+5n7/fffa/7www9zUzEoe4+uuuqqtD4zbNgwzQl2Z6WNNz0AACAINHoAAEAQaPQAAIAg5H1Mj12h2Y7NOOKII7xyEyZMSKROl19+eVqfOe200zT37ds363UK0QknnKD53HPP9c7Zvx/pTje37rzzTu/YXuOhhx4qUz1RdpUrV/aOr7jiCs29e/fWbFfHFvHv29KlSzXvs88+Xjk7puenn34qX2WRE82bN9dsV7QvjR37M23atKzXCT77/OVz6Yd84k0PAAAIAo0eAAAQhJx0b3311VdplatWrZrmG2+80TvXqVMnzUuWLMlOxVLYbbfdNB9wwAE5/S747H1+/PHH0/qM7ZpKl3OpF87N5Hoom65du3rHdtX2Sy+9VPOgQYPSul68O3zhwoWa58+fn0kVkWV169b1jgcOHJjW5958803Npa2Kj/KLDyM4++yzE6pJ/vDTHgAABIFGDwAACAKNHgAAEIScjOl57LHHNG+33XbeOTu11DryyCO945NPPlnz0KFDs1e5Etilzu3WGLvsskvKz4wYMSKndSpWdgyPiMjdd9+t2U4///XXX71y9h7ZHZq32mqrlN9lrxGfxmzHkxXC0umFyN6bm2++2Tv33HPPab7//vvTup7dgf2cc84pZ+2Qay+99JJ33LBhwxLLLV++3Du2yxSsXLky+xULnB1fFx9DZ5eW+OSTTzTHl4goZLzpAQAAQaDRAwAAgpCT7q3ffvtN8z333OOd69ixo+Z69eqlvMbFF1+s+fnnn/fO/fjjj+Wtosfu8ltalxYyY1dajk9LT9W1NH78eO/48MMP19ylSxfNpa2mfM0112geNWqUd85eA9ljd2l+//33Ndsp5SL+Srtr165N69pPPfWU5vhzOmDAgDLVE7nXqFEj7ziKohLL3Xfffd7xG2+8kbM6FTq7e0CTJk28c/Xr19e8//77e+fsDgJbbrllyut3795d86uvvqp51qxZZa9sBcWbHgAAEAQaPQAAIAg533B02bJl3rF95V1a99aee+6peccdd/TOpdu9ZUein3/++SnLnXrqqWldD+mJdx3ZGVpxdoaV7dLq1q1bWt81efJk79h2n5U2K8jOHrKrkrIid/mccsopmnfffXfNhx12mFdu8eLFaV3PbnrYrFkzzT///LNXrn///mWqJ3LDblgZXwXddm/ZVZfjM/uQmv1d+PDDD3vn7PMWZ38P2yEB8efG7qawww47ZFzPiow3PQAAIAg0egAAQBBo9AAAgCDkfExP3Icffqj5b3/7W1qfad68uXc8adIkzQcddFCJWcSf3nfdddeVqZ4lmTFjhuZc7/xeyK6//nrveLPNNktZtm/fvppvu+22tK7/3nvvaf7Xv/7lnbMrN5fGjglZtWpVWp/BX7PP9MyZMzV/8MEHaX1+m2228Y7vuusuzRts8Od/o8VXkk33viP7Bg8erNkuTxGfov7f//5Xs126JL76OlKbPn265viU9d122y3l5+yq119//XVW61Taz/eKiDc9AAAgCDR6AABAEPLevWU3D23VqpXmDh06pPzMvffeW+pxKvZ1eDY2ldxjjz0029e48amDIWratKlmuyGoiH8fKlWqVO7v+uKLL8p9DctOrbV1RdnZjYN79+6tec2aNSk/Yzd/HTlypHeuZs2amocMGaL5jjvuKFc9kbn4sg72Z2G8e9J68MEHNS9atCj7FQtMvFt+6tSpWb2+3aT5u+++887Z+9yuXTvNdrPxioqf8AAAIAg0egAAQBDy3r1l2U0C27dvn/Xr2y6tVJvdZcquDhtq91bjxo01226J+IZ22ehazDY7s8+u3F0R61qRtW7dOuW50aNHpzxnu8EeeOABzTvttJNXznZl2g1k7WwU5NdZZ53lHW+77bYllrMzjURK//uAisfufGBXahbxu7fGjRuXryplBW96AABAEGj0AACAINDoAQAAQUh0TE+u2fEAdkzPK6+84pWzO9DaabYo3T333KM5PhajorO7gbOzeuYWLlzoHdvVdf/5z39qji9jUKtWLc126m18Z2672q99TpFfl156qeazzz7bO5dqvGSbNm2842+//Tb7FUPiFixYkHQVyoQ3PQAAIAg0egAAQBAKsntr8eLFmu3maXYKvIjIsGHD0rre3nvvrZnurey78sork66CNGjQwDvu169fieXmzJnjHbMZYummTJniHV9wwQWabTfI5MmTvXL22bQrrE+YMMErZ6ezI7923HFHzeecc47m+Krlv/32m+aHHnpIM91Zxct2aca7uCs63vQAAIAg0OgBAABBSLR7a/bs2ZqfeOIJ79wuu+yiOb6y53333ac5/no9X4444gjN8RWIlyxZku/qVGh2Zc98sl1a8dVga9Soodm+nrWzukREvv/++xzVrjjZ59jm+Kysu+++W3Pt2rU1n3zyyV45uhfzp169et7xiy++qLl+/fopP3fXXXdpvuqqq7JfMZTbbrvtpjn++8pauXKlZvtz295jEX/DXzsT02YRkSpVqmi++eabNT/33HNeOft3Ldd40wMAAIJAowcAAASBRg8AAAhComN67E7J8Z17K7rtt99es92lOyR2nEZ8Gqv16KOPao6P3Sovu1t6/PrHH398ys99+eWXmo899ljNM2fOzGLt8IeWLVt6x3//+98133rrrZo//vjjvNUJvvi4ndLG8Vj5HI+BP8V/7+y6666a7RIDIiLnn3++ZjvOJm716tWaf/75Z81bbbVVys+MGDFC86JFi1LWsXr16pq/++47rxxjegAAALKMRg8AAAhCQa7InG1Lly7VbF+7bbPNNml9vm/fvt6xfZW4du3actau4rrllls0280l7WvMuLfeess7tit72mnl8W4mu6qz7VaLv+K1m4euWLFCc/wejRo1KuV3IfueeeYZ79iu1ptqdWzkV2ldGNa4ceO842nTpuWgNiiJXd7BLvsgInL66aeX+XrxzULtz+PPPvtMc3xF9fLK9jCHsuBNDwAACAKNHgAAEARnX2f9ZWHn0i9coA488EDNtgtExH+1WBrbvfPLL7+Uu05RFLm/LlU22b6XdnbOyJEjvXP230d8lte6devK/F32GvHPv/3225pTrQ6csIlRFO2X7YtWxGdzv/3+/Mf84IMPvHPdunXTPGTIkLzVKdsK4dlMV3yzXbvhqBXvRomvrlvAsv5sZvteXnbZZZrjG2yX5pVXXinxc++//75Xbs2aNeWoXYWS8l7ypgcAAASBRg8AAAgCjR4AABAExvSUYv/99/eOX3rpJc01a9ZM+bnWrVtrtmNMMlVo4wbsatUiIuedd57m6667zjuXyZgeuyv6u+++652zywUsW7aszNfOg6Ie07PJJptotuN44js7N27cWHM2xr0lpdCezbhGjRppjk9Ft1PYb7zxRs12t2wRf5pzgavwY3rq1q2r2S7xIeIvA2FXSRbxV8UPBGN6AABA2Gj0AACAILAicynimx9efvnlmnv06KHZTgcUEZkwYUJuK1bBzZ8/3zu+4YYbNNuNPkVEevbsqdlucDhjxgyv3J133ql59uzZmuNTLpGsrl27am7SpInmpk2beuUKuUurmDRr1kxz1apVU5ZbtWqV5iLqzio4dlkB+3whfbzpAQAAQaDRAwD4//buGLSSKgzD8PuxYJFWkIDRKCjiIlgI24gWFmHVRcFKwU6sFNZKtE1nZ2OnsoWoCDarhVoI2oiIxCLroixicG0WUSsLif4WCTERFt2ZJMd7zvs0d+Zyiw9+LnycmTMjDcHSI0mShuCW9QWw6NtidUDXW9b3v3F7/30g/3z8w/b29rFlOko9/Te3trYOnC8tLe0dr62t7R1vbGwcW6Zj9r/fsq7/zC3rkiRpbJYeSZI0BLesSzo0+5/iu76+vnfcy+Wsnq2urraOIB05V3okSdIQLD2SJGkIXt6SdGiWl5dbR5Ckq3KlR5IkDcHSI0mShmDpkSRJQ7D0SJKkIVh6JEnSECw9kiRpCNe6Zf0nYOtff6XDdFSPSXWWbTjPfjjLvhzFPJ1lG1ed5TW9ZV2SJGlReXlLkiQNwdIjSZKGMETpSXIiyUaS91tn0TxJTif5JsmlJC+0zqPpkpxNspnkQpLnWufRdEleT3IlyWbrLJqn91kOUXqAs8DF1iE0T5ITwCvAg8BJ4IkkJ9um0hRJ7gKeBk4BdwNnktzWNpVmOAecbh1Ch+IcHc+y+9KTZAV4GHi1dRbNdgq4VFXfVdXvwNvAo40zaZo7gc+r6req2gY+AR5rnEkTVdWnwM+tc2i+3mfZfekBXgaeB/5sHUSz3Qj8sO/88u53WjybwH1Jrk+yBDwE3NQ4k6TOdV16kpwBrlTVl62zSPpbVV0EXgI+Aj4AvgL+aBpKUve6Lj3AvcAjSb5n51LIA0neaBtJM/zIwdWAld3vtICq6rWquqeq7gd+Ab5tnUlS37ouPVX1YlWtVNUtwOPAx1X1ZONYmu4L4PYktya5jp2Znm+cSRMluWH382Z27ud5s20iSb3ruvSoL7s3vD4LfMjObrx3qupC21Sa4d0kXwPvAc9U1a+tA2maJG8BnwF3JLmc5KnWmTRN77P0NRSSJGkIrvRIkqQhWHokSdIQLD2SJGkIlh5JkjQES48kSRqCpUeSJA3B0iNJkoZg6ZEkSUP4C96sxdVha6m8AAAAAElFTkSuQmCC",
            "text/plain": [
              "<Figure size 720x720 with 25 Axes>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "# Show the first 25 images in the training dataset.\n",
        "plt.figure(figsize=(10,10))\n",
        "for i in range(25):\n",
        "  plt.subplot(5,5,i+1)\n",
        "  plt.xticks([])\n",
        "  plt.yticks([])\n",
        "  plt.imshow(train_images[i], cmap=plt.cm.gray)\n",
        "  plt.xlabel(train_labels[i])\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 23,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7PfmuhbgfdAA",
        "outputId": "f170fe92-c5ba-432e-ac7e-551b5dc86eee"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "(28, 28)"
            ]
          },
          "execution_count": 23,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "train_images[0].shape # 28px x 28px grayscale image"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 24,
      "metadata": {
        "id": "WxikfnvsN-xX"
      },
      "outputs": [],
      "source": [
        "train_images = train_images.astype(\"float\") / 225.0 # Normalizing the training data\n",
        "test_images = test_images.astype(\"float\") / 225.0 # Normalizing the testing data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "id": "KOzPObgUfEco"
      },
      "outputs": [],
      "source": [
        "first_model = keras.models.Sequential([ # build simple model and see accuracy\n",
        "        keras.layers.Flatten(),\n",
        "        keras.layers.Dense(128, activation = \"relu\", name = \"layer1\"),\n",
        "        keras.layers.Dense(128, activation = \"relu\", name = \"layer2\"),\n",
        "        keras.layers.Dense(128, activation = \"relu\", name = \"layer3\"),\n",
        "        keras.layers.Dense(128, activation = \"relu\", name = \"layer4\"),\n",
        "        \n",
        "        keras.layers.Dense(10, activation='softmax', name = \"output\"),\n",
        "])\n",
        "\n",
        "first_model.compile(loss='sparse_categorical_crossentropy', optimizer='adam', metrics=['accuracy'])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "p8KknegkiCop",
        "outputId": "4a6d2685-3969-4518-d4e1-61c5e443fa07"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 1/10\n",
            "1875/1875 [==============================] - 10s 5ms/step - loss: 0.2313 - accuracy: 0.9310\n",
            "Epoch 2/10\n",
            "1875/1875 [==============================] - 9s 5ms/step - loss: 0.1127 - accuracy: 0.9655\n",
            "Epoch 3/10\n",
            "1875/1875 [==============================] - 10s 5ms/step - loss: 0.0842 - accuracy: 0.9736\n",
            "Epoch 4/10\n",
            "1875/1875 [==============================] - 9s 5ms/step - loss: 0.0694 - accuracy: 0.9792\n",
            "Epoch 5/10\n",
            "1875/1875 [==============================] - 9s 5ms/step - loss: 0.0611 - accuracy: 0.9817\n",
            "Epoch 6/10\n",
            "1875/1875 [==============================] - 9s 5ms/step - loss: 0.0494 - accuracy: 0.9847\n",
            "Epoch 7/10\n",
            "1875/1875 [==============================] - 9s 5ms/step - loss: 0.0447 - accuracy: 0.9863\n",
            "Epoch 8/10\n",
            "1875/1875 [==============================] - 9s 5ms/step - loss: 0.0409 - accuracy: 0.9874\n",
            "Epoch 9/10\n",
            "1875/1875 [==============================] - 10s 5ms/step - loss: 0.0355 - accuracy: 0.9898\n",
            "Epoch 10/10\n",
            "1875/1875 [==============================] - 9s 5ms/step - loss: 0.0323 - accuracy: 0.9904\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "<keras.callbacks.History at 0x7fef01596590>"
            ]
          },
          "execution_count": 11,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "first_model.fit(train_images, train_labels, \n",
        "          epochs = 10, \n",
        "          batch_size=32)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AFr5iaGV6CF8",
        "outputId": "04f87270-d262-4e64-95d1-1b5c877d0489"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "313/313 [==============================] - 1s 2ms/step - loss: 0.1313 - accuracy: 0.9735\n"
          ]
        }
      ],
      "source": [
        "model1_eval = first_model.evaluate(test_images, test_labels, return_dict=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 25,
      "metadata": {
        "id": "qTUKWwEY7MHQ"
      },
      "outputs": [],
      "source": [
        "def model_builder(hp: kt.HyperParameters) -> keras.models.Sequential:\n",
        "    \"\"\"function that take HyperParameters object to make diffreant values for hyperparameter optimization\n",
        "\n",
        "    Args:\n",
        "        hp (kt.HyperParameters): object contean function that allwod to make diffreant values\n",
        "\n",
        "    Returns:\n",
        "        _type_: keras.models.Sequential\n",
        "    \"\"\"\n",
        "    hp_units = hp.Int('units', min_value=128, max_value=512, step=32)\n",
        "    hp_optimizer = hp.Choice('optimizer', values=['adam','adamax'])\n",
        "    activations=hp.Choice(\"activation\", [\"relu\", \"tanh\"])\n",
        "\n",
        "    model_hp = keras.models.Sequential([\n",
        "                                    \n",
        "    keras.layers.Flatten(input_shape=(28, 28)),\n",
        "    keras.layers.Dense(units=hp_units, activation =activations, name = \"layer1\"),\n",
        "    Dropout(0.25),\n",
        "    keras.layers.Dense(units=hp_units, activation =activations, name = \"layer2\"),\n",
        "    Dropout(0.25),\n",
        "    keras.layers.Dense(units=hp_units, activation =activations, name = \"layer3\"),\n",
        "    Dropout(0.25),\n",
        "    keras.layers.Dense(units=hp_units, activation =activations, name = \"layer4\"),\n",
        "    Dropout(0.25),\n",
        "    \n",
        "    keras.layers.Dense(10, activation='softmax', name = \"output\"),\n",
        "    ])\n",
        "\n",
        "    model_hp.compile(loss='sparse_categorical_crossentropy', \n",
        "                    optimizer=hp_optimizer, \n",
        "                    metrics=['accuracy'])\n",
        "    \n",
        "    return model_hp\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 26,
      "metadata": {
        "id": "JPsA2yqn0ADp"
      },
      "outputs": [],
      "source": [
        "tuner = kt.RandomSearch(\n",
        "    hypermodel=model_builder,\n",
        "    objective=\"val_accuracy\",\n",
        "    max_trials=10,\n",
        "    executions_per_trial=3,\n",
        "    overwrite=True,\n",
        "    project_name=\"khyperband\",\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 27,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Rd_sKf-u0rLo",
        "outputId": "4381b19c-0195-4b59-a031-19ef9aada10d"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Search space summary\n",
            "Default search space size: 3\n",
            "units (Int)\n",
            "{'default': None, 'conditions': [], 'min_value': 128, 'max_value': 512, 'step': 32, 'sampling': None}\n",
            "optimizer (Choice)\n",
            "{'default': 'adam', 'conditions': [], 'values': ['adam', 'adamax'], 'ordered': False}\n",
            "activation (Choice)\n",
            "{'default': 'relu', 'conditions': [], 'values': ['relu', 'tanh'], 'ordered': False}\n"
          ]
        }
      ],
      "source": [
        "tuner.search_space_summary()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 28,
      "metadata": {
        "id": "eCf_1BTH1Jkf"
      },
      "outputs": [],
      "source": [
        "x_train = train_images[:-10000]\n",
        "x_val = train_images[-10000:]\n",
        "y_train = train_labels[:-10000]\n",
        "y_val = train_labels[-10000:]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "split data to train and validation "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 29,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NDxgqU2d1Jm3",
        "outputId": "b756aebe-78df-455a-c3e0-afbff5367f35"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Trial 10 Complete [00h 08m 01s]\n",
            "val_accuracy: 0.975000003973643\n",
            "\n",
            "Best val_accuracy So Far: 0.9815333485603333\n",
            "Total elapsed time: 01h 29m 55s\n",
            "INFO:tensorflow:Oracle triggered exit\n"
          ]
        }
      ],
      "source": [
        "tuner.search(x_train, y_train, \n",
        "             epochs=10, \n",
        "             validation_data=(x_val, y_val),\n",
        "             callbacks=[EarlyStopping(monitor='val_accuracy', patience=3)])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 30,
      "metadata": {
        "id": "QgVxGkBf1Jpc"
      },
      "outputs": [],
      "source": [
        "models = tuner.get_best_models(num_models=2)\n",
        "best_model = models[0]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "take the best model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 34,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MKaxhCdHzk9e",
        "outputId": "07512f21-38a1-4398-c40c-2c3783ee002f"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Model: \"sequential\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " flatten (Flatten)           (None, 784)               0         \n",
            "                                                                 \n",
            " layer1 (Dense)              (None, 448)               351680    \n",
            "                                                                 \n",
            " dropout (Dropout)           (None, 448)               0         \n",
            "                                                                 \n",
            " layer2 (Dense)              (None, 448)               201152    \n",
            "                                                                 \n",
            " dropout_1 (Dropout)         (None, 448)               0         \n",
            "                                                                 \n",
            " layer3 (Dense)              (None, 448)               201152    \n",
            "                                                                 \n",
            " dropout_2 (Dropout)         (None, 448)               0         \n",
            "                                                                 \n",
            " layer4 (Dense)              (None, 448)               201152    \n",
            "                                                                 \n",
            " dropout_3 (Dropout)         (None, 448)               0         \n",
            "                                                                 \n",
            " output (Dense)              (None, 10)                4490      \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 959,626\n",
            "Trainable params: 959,626\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ]
        }
      ],
      "source": [
        "best_model.summary()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 31,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WI3oLv145V12",
        "outputId": "09e64344-531c-48c9-ce43-8fa02b3ed3bd"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "313/313 [==============================] - 2s 5ms/step - loss: 0.0634 - accuracy: 0.9840\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "{'accuracy': 0.984000027179718, 'loss': 0.06342479586601257}"
            ]
          },
          "execution_count": 31,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "best_model.evaluate(test_images, test_labels, return_dict=True)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "good accuracy and loss"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 32,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EBt8_G-X6EkH",
        "outputId": "5df52f22-a9f5-4ee1-80ba-231b0c4f4d71"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "{'accuracy': 0.9735000133514404, 'loss': 0.13132120668888092}"
            ]
          },
          "execution_count": 32,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "model1_eval"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 33,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jLxIq_ZR4hCK",
        "outputId": "21d8d1b9-175a-46cf-c3c6-e0092ea2579f"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Assets written to: /tmp/tmp7v9mvlsp/assets\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:absl:Buffer deduplication procedure will be skipped when flatbuffer library is not properly loaded\n"
          ]
        }
      ],
      "source": [
        "# Convert the model.\n",
        "converter = tf.lite.TFLiteConverter.from_keras_model(best_model)\n",
        "tflite_model = converter.convert()\n",
        "\n",
        "# Save the model.\n",
        "with open('detect.tflite', 'wb') as f:\n",
        "  f.write(tflite_model)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "U9G61bf88Udf"
      },
      "outputs": [],
      "source": []
    }
  ],
  "metadata": {
    "colab": {
      "collapsed_sections": [],
      "name": "FFN_digit_recognition.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.2"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}