ML_Project_Testing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "sQykkQi0QxXh"
   },
   "outputs": [],
   "source": [
    "#import everything\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras.optimizers import Adam\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "from tensorflow.keras.preprocessing import image_dataset_from_directory\n",
    "from keras.utils.np_utils import to_categorical\n",
    "from keras.callbacks import ModelCheckpoint\n",
    "from keras.models import load_model, Sequential\n",
    "from keras.layers import Dense, Dropout, Activation, Flatten, Conv2D, MaxPooling2D, BatchNormalization\n",
    "import tensorflow_datasets as tfds\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib import cm\n",
    "import numpy as np\n",
    "from PIL import Image, ImageTk, ImageDraw, ImageEnhance, ImageGrab\n",
    "import PIL.ImageOps\n",
    "import random\n",
    "import time\n",
    "import math\n",
    "import heapq\n",
    "import cv2\n",
    "import PIL\n",
    "import sys\n",
    "import os\n",
    "from time import sleep\n",
    "from imutils import contours\n",
    "import imutils\n",
    "from tkinter import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "IaZT0rP4Qxub"
   },
   "outputs": [],
   "source": [
    "# Do not run - load datasets and normalize data to match MNIST data parameters. \n",
    "# only do this once to create dataset\n",
    "#mnist_corr_shear  = tfds.as_numpy(tfds.load(name=\"mnist_corrupted/shear\", split=None, shuffle_files=True))\n",
    "#mnist_corr_rotate = tfds.as_numpy(tfds.load(name=\"mnist_corrupted/rotate\", split=None, shuffle_files=True))\n",
    "#mnist_corr_bright = tfds.as_numpy(tfds.load(name=\"mnist_corrupted/brightness\", split=None, shuffle_files=True))\n",
    "\n",
    "#convertPath = 'D:/Documents/Machine Learning/Project Files/mnist_png/mnist_c/shear/'\n",
    "\n",
    "#for count, value in enumerate(mnist_corr_shear['train']):\n",
    "#\tplt.imshow(255-value['image'], cmap=cm.binary)\n",
    "#\t\n",
    "#\t#set plot x axis label to be the image number\n",
    "#\tplt.xlabel(\"Label = \" + str(value['label']))\n",
    "#\tplt.show()\n",
    "#\timg2 = np.zeros((28,28,3), np.uint8)\n",
    "#\timg2[:,:,0] = value['image'][:,:,0]\n",
    "#\timg2[:,:,1] = value['image'][:,:,0]\n",
    "#\timg2[:,:,2] = value['image'][:,:,0]\n",
    "#\t#red, green, blue = img2[:,:,0], img2[:,:,1], img2[:,:,2]\n",
    "#\t#mask = (red == 126) & (green == 126) & (blue == 126)\n",
    "#\t#img2[:,:,:3][mask] = [0, 0, 0]\n",
    "\n",
    " \n",
    "#\t#plt.imsave(f\"{convertPath}{str(value['label'])}/{str(count+20000)}.png\", img2, format='png')\n",
    "#\tif count == 3:\n",
    "#\t\tbreak\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print images from dataset to see if they look correct\n",
    "#corr_train = mnist_corr_shear['train']\n",
    "#corr_test = mnist_corr_shear['test']\n",
    "\n",
    "#fig, axes = plt.subplots(3, 3, figsize=(8, 8))\n",
    "#fig.subplots_adjust(hspace=0.2, wspace=0.1)\n",
    "\n",
    "#for i, (elem, ax) in enumerate(zip(corr_train, axes.flat)):\n",
    "#    image = tf.squeeze(elem['image'])\n",
    "#    label = elem['label']\n",
    "#    \n",
    "#    ax.imshow(image, cmap='gray')\n",
    "#    ax.set_xticklabels([])\n",
    "#    ax.set_yticklabels([])\n",
    "#    ax.text(0.7, -0.12, f'Digit = {label}', ha='right', transform=ax.transAxes, color='black')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 71322 files belonging to 13 classes.\n",
      "Using 49926 files for training.\n",
      "Found 71322 files belonging to 13 classes.\n",
      "Using 21396 files for validation.\n",
      "Found 71322 files belonging to 13 classes.\n",
      "Using 70608 files for validation.\n"
     ]
    }
   ],
   "source": [
    "# setting up tf ImageDataGenerator parameters\n",
    "batch_size = 10\n",
    "num_classes = 13\n",
    "epochs = 20\n",
    "class_names = ['0','1','2','3','4','5','6','7','8','9', 'add', 'sub', 'mult']\n",
    "\n",
    "datasetpath = 'D:/Documents/Machine Learning/Project Files/mnist_png/_Final/'\n",
    "\n",
    "ds_train      = image_dataset_from_directory(datasetpath, validation_split=0.3, image_size=(28, 28), label_mode='categorical', class_names=class_names, color_mode='grayscale', batch_size=batch_size, seed=134, subset='training')\n",
    "ds_validation = image_dataset_from_directory(datasetpath, validation_split=0.3, image_size=(28, 28), label_mode='categorical', class_names=class_names, color_mode='grayscale', batch_size=batch_size, seed=134, subset='validation')\n",
    "ds_test       = image_dataset_from_directory(datasetpath, validation_split=0.99, image_size=(28, 28), label_mode='categorical', class_names=class_names, color_mode='grayscale', batch_size=batch_size, shuffle=False, subset='validation')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def base_cnn():\n",
    "    model=Sequential()\n",
    "    model.add(Conv2D(filters=64, kernel_size = (3,3), activation=\"relu\", input_shape=(28,28,1)))\n",
    "    model.add(Conv2D(filters=64, kernel_size = (3,3), activation=\"relu\"))\n",
    "    model.add(MaxPooling2D(pool_size=(2,2)))\n",
    "    model.add(BatchNormalization())\n",
    "\n",
    "    model.add(Conv2D(filters=128, kernel_size = (3,3), activation=\"relu\"))\n",
    "    model.add(Conv2D(filters=128, kernel_size = (3,3), activation=\"relu\"))\n",
    "    model.add(MaxPooling2D(pool_size=(2,2)))\n",
    "    model.add(BatchNormalization())    \n",
    "\n",
    "    model.add(Conv2D(filters=256, kernel_size = (3,3), activation=\"relu\"))\n",
    "    model.add(MaxPooling2D(pool_size=(2,2)))\n",
    "    model.add(BatchNormalization())\n",
    "    \n",
    "    model.add(Flatten())\n",
    "    model.add(Dense(512,activation=\"relu\"))\n",
    "    \n",
    "    model.add(Dense(13,activation=\"softmax\"))\n",
    "    \n",
    "    model.compile(loss=\"categorical_crossentropy\", optimizer=\"adam\", metrics=[\"accuracy\"])\n",
    "    #print(model.summary())\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Train the model\n",
    "# create a callback that will save the best model while training\n",
    "history_activations = []\n",
    "model = base_cnn()\n",
    "save_best_model = ModelCheckpoint(filepath='best_model', monitor='val_accuracy', mode='max', save_best_only=True, verbose=1)\n",
    "history_activations.append(model.fit(ds_train, epochs=epochs, verbose=1, validation_data=ds_validation, callbacks=[save_best_model])) \n",
    "\n",
    "plt.plot(history_activations[0].history['accuracy'], 'o-', label='acc')\n",
    "plt.plot(history_activations[0].history['val_accuracy'], 'o-', label='val_acc')\n",
    "plt.ylabel('accuracy')\n",
    "plt.legend(loc='lower right')\n",
    "plt.show()\n",
    "\n",
    "# Evaluate the best model saved (i.e., model with best validation accuracy) on the test set\n",
    "saved_model = load_model(filepath='best_model')\n",
    "print('Test accuracy for relu activation: {}'.format(saved_model.evaluate(ds_test, verbose=1)[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 645/7061 [=>............................] - ETA: 31s - loss: 0.0047 - accuracy: 0.9991"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32md:\\Documents\\Machine Learning\\Project Files\\ML_Project_Testing.ipynb Cell 7'\u001b[0m in \u001b[0;36m<cell line: 14>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     <a href='vscode-notebook-cell:/d%3A/Documents/Machine%20Learning/Project%20Files/ML_Project_Testing.ipynb#ch0000006?line=10'>11</a>\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39mtensorflow\u001b[39;00m\u001b[39m.\u001b[39;00m\u001b[39mkeras\u001b[39;00m \u001b[39mimport\u001b[39;00m layers\n\u001b[0;32m     <a href='vscode-notebook-cell:/d%3A/Documents/Machine%20Learning/Project%20Files/ML_Project_Testing.ipynb#ch0000006?line=11'>12</a>\u001b[0m tf\u001b[39m.\u001b[39mkeras\u001b[39m.\u001b[39mutils\u001b[39m.\u001b[39mplot_model(model, to_file\u001b[39m=\u001b[39m\u001b[39m'\u001b[39m\u001b[39mmodel_summary.png\u001b[39m\u001b[39m'\u001b[39m, show_shapes\u001b[39m=\u001b[39m\u001b[39mTrue\u001b[39;00m, expand_nested\u001b[39m=\u001b[39m\u001b[39mTrue\u001b[39;00m)\n\u001b[1;32m---> <a href='vscode-notebook-cell:/d%3A/Documents/Machine%20Learning/Project%20Files/ML_Project_Testing.ipynb#ch0000006?line=13'>14</a>\u001b[0m loss, acc \u001b[39m=\u001b[39m model\u001b[39m.\u001b[39;49mevaluate(ds_test, verbose\u001b[39m=\u001b[39;49m\u001b[39m1\u001b[39;49m)\n\u001b[0;32m     <a href='vscode-notebook-cell:/d%3A/Documents/Machine%20Learning/Project%20Files/ML_Project_Testing.ipynb#ch0000006?line=14'>15</a>\u001b[0m \u001b[39mprint\u001b[39m(\u001b[39m\"\u001b[39m\u001b[39mRestored model, accuracy: \u001b[39m\u001b[39m{:5.2f}\u001b[39;00m\u001b[39m%\u001b[39m\u001b[39m\"\u001b[39m\u001b[39m.\u001b[39mformat(\u001b[39m100\u001b[39m \u001b[39m*\u001b[39m acc))\n",
      "File \u001b[1;32mC:\\Program Files\\Python310\\lib\\site-packages\\keras\\utils\\traceback_utils.py:64\u001b[0m, in \u001b[0;36mfilter_traceback.<locals>.error_handler\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m     <a href='file:///c%3A/Program%20Files/Python310/lib/site-packages/keras/utils/traceback_utils.py?line=61'>62</a>\u001b[0m filtered_tb \u001b[39m=\u001b[39m \u001b[39mNone\u001b[39;00m\n\u001b[0;32m     <a href='file:///c%3A/Program%20Files/Python310/lib/site-packages/keras/utils/traceback_utils.py?line=62'>63</a>\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m---> <a href='file:///c%3A/Program%20Files/Python310/lib/site-packages/keras/utils/traceback_utils.py?line=63'>64</a>\u001b[0m   \u001b[39mreturn\u001b[39;00m fn(\u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)\n\u001b[0;32m     <a href='file:///c%3A/Program%20Files/Python310/lib/site-packages/keras/utils/traceback_utils.py?line=64'>65</a>\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mException\u001b[39;00m \u001b[39mas\u001b[39;00m e:  \u001b[39m# pylint: disable=broad-except\u001b[39;00m\n\u001b[0;32m     <a href='file:///c%3A/Program%20Files/Python310/lib/site-packages/keras/utils/traceback_utils.py?line=65'>66</a>\u001b[0m   filtered_tb \u001b[39m=\u001b[39m _process_traceback_frames(e\u001b[39m.\u001b[39m__traceback__)\n",
      "File \u001b[1;32mC:\\Program Files\\Python310\\lib\\site-packages\\keras\\engine\\training.py:1716\u001b[0m, in \u001b[0;36mModel.evaluate\u001b[1;34m(self, x, y, batch_size, verbose, sample_weight, steps, callbacks, max_queue_size, workers, use_multiprocessing, return_dict, **kwargs)\u001b[0m\n\u001b[0;32m   <a href='file:///c%3A/Program%20Files/Python310/lib/site-packages/keras/engine/training.py?line=1713'>1714</a>\u001b[0m \u001b[39mwith\u001b[39;00m tf\u001b[39m.\u001b[39mprofiler\u001b[39m.\u001b[39mexperimental\u001b[39m.\u001b[39mTrace(\u001b[39m'\u001b[39m\u001b[39mtest\u001b[39m\u001b[39m'\u001b[39m, step_num\u001b[39m=\u001b[39mstep, _r\u001b[39m=\u001b[39m\u001b[39m1\u001b[39m):\n\u001b[0;32m   <a href='file:///c%3A/Program%20Files/Python310/lib/site-packages/keras/engine/training.py?line=1714'>1715</a>\u001b[0m   callbacks\u001b[39m.\u001b[39mon_test_batch_begin(step)\n\u001b[1;32m-> <a href='file:///c%3A/Program%20Files/Python310/lib/site-packages/keras/engine/training.py?line=1715'>1716</a>\u001b[0m   tmp_logs \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mtest_function(iterator)\n\u001b[0;32m   <a href='file:///c%3A/Program%20Files/Python310/lib/site-packages/keras/engine/training.py?line=1716'>1717</a>\u001b[0m   \u001b[39mif\u001b[39;00m data_handler\u001b[39m.\u001b[39mshould_sync:\n\u001b[0;32m   <a href='file:///c%3A/Program%20Files/Python310/lib/site-packages/keras/engine/training.py?line=1717'>1718</a>\u001b[0m     context\u001b[39m.\u001b[39masync_wait()\n",
      "File \u001b[1;32m~\\AppData\\Roaming\\Python\\Python310\\site-packages\\tensorflow\\python\\util\\traceback_utils.py:150\u001b[0m, in \u001b[0;36mfilter_traceback.<locals>.error_handler\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m    <a href='file:///c%3A/Users/Saksham%20Goyal/AppData/Roaming/Python/Python310/site-packages/tensorflow/python/util/traceback_utils.py?line=147'>148</a>\u001b[0m filtered_tb \u001b[39m=\u001b[39m \u001b[39mNone\u001b[39;00m\n\u001b[0;32m    <a href='file:///c%3A/Users/Saksham%20Goyal/AppData/Roaming/Python/Python310/site-packages/tensorflow/python/util/traceback_utils.py?line=148'>149</a>\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m--> <a href='file:///c%3A/Users/Saksham%20Goyal/AppData/Roaming/Python/Python310/site-packages/tensorflow/python/util/traceback_utils.py?line=149'>150</a>\u001b[0m   \u001b[39mreturn\u001b[39;00m fn(\u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)\n\u001b[0;32m    <a href='file:///c%3A/Users/Saksham%20Goyal/AppData/Roaming/Python/Python310/site-packages/tensorflow/python/util/traceback_utils.py?line=150'>151</a>\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mException\u001b[39;00m \u001b[39mas\u001b[39;00m e:\n\u001b[0;32m    <a href='file:///c%3A/Users/Saksham%20Goyal/AppData/Roaming/Python/Python310/site-packages/tensorflow/python/util/traceback_utils.py?line=151'>152</a>\u001b[0m   filtered_tb \u001b[39m=\u001b[39m _process_traceback_frames(e\u001b[39m.\u001b[39m__traceback__)\n",
      "File \u001b[1;32m~\\AppData\\Roaming\\Python\\Python310\\site-packages\\tensorflow\\python\\eager\\def_function.py:915\u001b[0m, in \u001b[0;36mFunction.__call__\u001b[1;34m(self, *args, **kwds)\u001b[0m\n\u001b[0;32m    <a href='file:///c%3A/Users/Saksham%20Goyal/AppData/Roaming/Python/Python310/site-packages/tensorflow/python/eager/def_function.py?line=911'>912</a>\u001b[0m compiler \u001b[39m=\u001b[39m \u001b[39m\"\u001b[39m\u001b[39mxla\u001b[39m\u001b[39m\"\u001b[39m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_jit_compile \u001b[39melse\u001b[39;00m \u001b[39m\"\u001b[39m\u001b[39mnonXla\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[0;32m    <a href='file:///c%3A/Users/Saksham%20Goyal/AppData/Roaming/Python/Python310/site-packages/tensorflow/python/eager/def_function.py?line=913'>914</a>\u001b[0m \u001b[39mwith\u001b[39;00m OptionalXlaContext(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_jit_compile):\n\u001b[1;32m--> <a href='file:///c%3A/Users/Saksham%20Goyal/AppData/Roaming/Python/Python310/site-packages/tensorflow/python/eager/def_function.py?line=914'>915</a>\u001b[0m   result \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_call(\u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwds)\n\u001b[0;32m    <a href='file:///c%3A/Users/Saksham%20Goyal/AppData/Roaming/Python/Python310/site-packages/tensorflow/python/eager/def_function.py?line=916'>917</a>\u001b[0m new_tracing_count \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mexperimental_get_tracing_count()\n\u001b[0;32m    <a href='file:///c%3A/Users/Saksham%20Goyal/AppData/Roaming/Python/Python310/site-packages/tensorflow/python/eager/def_function.py?line=917'>918</a>\u001b[0m without_tracing \u001b[39m=\u001b[39m (tracing_count \u001b[39m==\u001b[39m new_tracing_count)\n",
      "File \u001b[1;32m~\\AppData\\Roaming\\Python\\Python310\\site-packages\\tensorflow\\python\\eager\\def_function.py:947\u001b[0m, in \u001b[0;36mFunction._call\u001b[1;34m(self, *args, **kwds)\u001b[0m\n\u001b[0;32m    <a href='file:///c%3A/Users/Saksham%20Goyal/AppData/Roaming/Python/Python310/site-packages/tensorflow/python/eager/def_function.py?line=943'>944</a>\u001b[0m   \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_lock\u001b[39m.\u001b[39mrelease()\n\u001b[0;32m    <a href='file:///c%3A/Users/Saksham%20Goyal/AppData/Roaming/Python/Python310/site-packages/tensorflow/python/eager/def_function.py?line=944'>945</a>\u001b[0m   \u001b[39m# In this case we have created variables on the first call, so we run the\u001b[39;00m\n\u001b[0;32m    <a href='file:///c%3A/Users/Saksham%20Goyal/AppData/Roaming/Python/Python310/site-packages/tensorflow/python/eager/def_function.py?line=945'>946</a>\u001b[0m   \u001b[39m# defunned version which is guaranteed to never create variables.\u001b[39;00m\n\u001b[1;32m--> <a href='file:///c%3A/Users/Saksham%20Goyal/AppData/Roaming/Python/Python310/site-packages/tensorflow/python/eager/def_function.py?line=946'>947</a>\u001b[0m   \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_stateless_fn(\u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwds)  \u001b[39m# pylint: disable=not-callable\u001b[39;00m\n\u001b[0;32m    <a href='file:///c%3A/Users/Saksham%20Goyal/AppData/Roaming/Python/Python310/site-packages/tensorflow/python/eager/def_function.py?line=947'>948</a>\u001b[0m \u001b[39melif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_stateful_fn \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[0;32m    <a href='file:///c%3A/Users/Saksham%20Goyal/AppData/Roaming/Python/Python310/site-packages/tensorflow/python/eager/def_function.py?line=948'>949</a>\u001b[0m   \u001b[39m# Release the lock early so that multiple threads can perform the call\u001b[39;00m\n\u001b[0;32m    <a href='file:///c%3A/Users/Saksham%20Goyal/AppData/Roaming/Python/Python310/site-packages/tensorflow/python/eager/def_function.py?line=949'>950</a>\u001b[0m   \u001b[39m# in parallel.\u001b[39;00m\n\u001b[0;32m    <a href='file:///c%3A/Users/Saksham%20Goyal/AppData/Roaming/Python/Python310/site-packages/tensorflow/python/eager/def_function.py?line=950'>951</a>\u001b[0m   \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_lock\u001b[39m.\u001b[39mrelease()\n",
      "File \u001b[1;32m~\\AppData\\Roaming\\Python\\Python310\\site-packages\\tensorflow\\python\\eager\\function.py:2956\u001b[0m, in \u001b[0;36mFunction.__call__\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   <a href='file:///c%3A/Users/Saksham%20Goyal/AppData/Roaming/Python/Python310/site-packages/tensorflow/python/eager/function.py?line=2952'>2953</a>\u001b[0m \u001b[39mwith\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_lock:\n\u001b[0;32m   <a href='file:///c%3A/Users/Saksham%20Goyal/AppData/Roaming/Python/Python310/site-packages/tensorflow/python/eager/function.py?line=2953'>2954</a>\u001b[0m   (graph_function,\n\u001b[0;32m   <a href='file:///c%3A/Users/Saksham%20Goyal/AppData/Roaming/Python/Python310/site-packages/tensorflow/python/eager/function.py?line=2954'>2955</a>\u001b[0m    filtered_flat_args) \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_maybe_define_function(args, kwargs)\n\u001b[1;32m-> <a href='file:///c%3A/Users/Saksham%20Goyal/AppData/Roaming/Python/Python310/site-packages/tensorflow/python/eager/function.py?line=2955'>2956</a>\u001b[0m \u001b[39mreturn\u001b[39;00m graph_function\u001b[39m.\u001b[39;49m_call_flat(\n\u001b[0;32m   <a href='file:///c%3A/Users/Saksham%20Goyal/AppData/Roaming/Python/Python310/site-packages/tensorflow/python/eager/function.py?line=2956'>2957</a>\u001b[0m     filtered_flat_args, captured_inputs\u001b[39m=\u001b[39;49mgraph_function\u001b[39m.\u001b[39;49mcaptured_inputs)\n",
      "File \u001b[1;32m~\\AppData\\Roaming\\Python\\Python310\\site-packages\\tensorflow\\python\\eager\\function.py:1853\u001b[0m, in \u001b[0;36mConcreteFunction._call_flat\u001b[1;34m(self, args, captured_inputs, cancellation_manager)\u001b[0m\n\u001b[0;32m   <a href='file:///c%3A/Users/Saksham%20Goyal/AppData/Roaming/Python/Python310/site-packages/tensorflow/python/eager/function.py?line=1848'>1849</a>\u001b[0m possible_gradient_type \u001b[39m=\u001b[39m gradients_util\u001b[39m.\u001b[39mPossibleTapeGradientTypes(args)\n\u001b[0;32m   <a href='file:///c%3A/Users/Saksham%20Goyal/AppData/Roaming/Python/Python310/site-packages/tensorflow/python/eager/function.py?line=1849'>1850</a>\u001b[0m \u001b[39mif\u001b[39;00m (possible_gradient_type \u001b[39m==\u001b[39m gradients_util\u001b[39m.\u001b[39mPOSSIBLE_GRADIENT_TYPES_NONE\n\u001b[0;32m   <a href='file:///c%3A/Users/Saksham%20Goyal/AppData/Roaming/Python/Python310/site-packages/tensorflow/python/eager/function.py?line=1850'>1851</a>\u001b[0m     \u001b[39mand\u001b[39;00m executing_eagerly):\n\u001b[0;32m   <a href='file:///c%3A/Users/Saksham%20Goyal/AppData/Roaming/Python/Python310/site-packages/tensorflow/python/eager/function.py?line=1851'>1852</a>\u001b[0m   \u001b[39m# No tape is watching; skip to running the function.\u001b[39;00m\n\u001b[1;32m-> <a href='file:///c%3A/Users/Saksham%20Goyal/AppData/Roaming/Python/Python310/site-packages/tensorflow/python/eager/function.py?line=1852'>1853</a>\u001b[0m   \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_build_call_outputs(\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_inference_function\u001b[39m.\u001b[39;49mcall(\n\u001b[0;32m   <a href='file:///c%3A/Users/Saksham%20Goyal/AppData/Roaming/Python/Python310/site-packages/tensorflow/python/eager/function.py?line=1853'>1854</a>\u001b[0m       ctx, args, cancellation_manager\u001b[39m=\u001b[39;49mcancellation_manager))\n\u001b[0;32m   <a href='file:///c%3A/Users/Saksham%20Goyal/AppData/Roaming/Python/Python310/site-packages/tensorflow/python/eager/function.py?line=1854'>1855</a>\u001b[0m forward_backward \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_select_forward_and_backward_functions(\n\u001b[0;32m   <a href='file:///c%3A/Users/Saksham%20Goyal/AppData/Roaming/Python/Python310/site-packages/tensorflow/python/eager/function.py?line=1855'>1856</a>\u001b[0m     args,\n\u001b[0;32m   <a href='file:///c%3A/Users/Saksham%20Goyal/AppData/Roaming/Python/Python310/site-packages/tensorflow/python/eager/function.py?line=1856'>1857</a>\u001b[0m     possible_gradient_type,\n\u001b[0;32m   <a href='file:///c%3A/Users/Saksham%20Goyal/AppData/Roaming/Python/Python310/site-packages/tensorflow/python/eager/function.py?line=1857'>1858</a>\u001b[0m     executing_eagerly)\n\u001b[0;32m   <a href='file:///c%3A/Users/Saksham%20Goyal/AppData/Roaming/Python/Python310/site-packages/tensorflow/python/eager/function.py?line=1858'>1859</a>\u001b[0m forward_function, args_with_tangents \u001b[39m=\u001b[39m forward_backward\u001b[39m.\u001b[39mforward()\n",
      "File \u001b[1;32m~\\AppData\\Roaming\\Python\\Python310\\site-packages\\tensorflow\\python\\eager\\function.py:499\u001b[0m, in \u001b[0;36m_EagerDefinedFunction.call\u001b[1;34m(self, ctx, args, cancellation_manager)\u001b[0m\n\u001b[0;32m    <a href='file:///c%3A/Users/Saksham%20Goyal/AppData/Roaming/Python/Python310/site-packages/tensorflow/python/eager/function.py?line=496'>497</a>\u001b[0m \u001b[39mwith\u001b[39;00m _InterpolateFunctionError(\u001b[39mself\u001b[39m):\n\u001b[0;32m    <a href='file:///c%3A/Users/Saksham%20Goyal/AppData/Roaming/Python/Python310/site-packages/tensorflow/python/eager/function.py?line=497'>498</a>\u001b[0m   \u001b[39mif\u001b[39;00m cancellation_manager \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[1;32m--> <a href='file:///c%3A/Users/Saksham%20Goyal/AppData/Roaming/Python/Python310/site-packages/tensorflow/python/eager/function.py?line=498'>499</a>\u001b[0m     outputs \u001b[39m=\u001b[39m execute\u001b[39m.\u001b[39;49mexecute(\n\u001b[0;32m    <a href='file:///c%3A/Users/Saksham%20Goyal/AppData/Roaming/Python/Python310/site-packages/tensorflow/python/eager/function.py?line=499'>500</a>\u001b[0m         \u001b[39mstr\u001b[39;49m(\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49msignature\u001b[39m.\u001b[39;49mname),\n\u001b[0;32m    <a href='file:///c%3A/Users/Saksham%20Goyal/AppData/Roaming/Python/Python310/site-packages/tensorflow/python/eager/function.py?line=500'>501</a>\u001b[0m         num_outputs\u001b[39m=\u001b[39;49m\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_num_outputs,\n\u001b[0;32m    <a href='file:///c%3A/Users/Saksham%20Goyal/AppData/Roaming/Python/Python310/site-packages/tensorflow/python/eager/function.py?line=501'>502</a>\u001b[0m         inputs\u001b[39m=\u001b[39;49margs,\n\u001b[0;32m    <a href='file:///c%3A/Users/Saksham%20Goyal/AppData/Roaming/Python/Python310/site-packages/tensorflow/python/eager/function.py?line=502'>503</a>\u001b[0m         attrs\u001b[39m=\u001b[39;49mattrs,\n\u001b[0;32m    <a href='file:///c%3A/Users/Saksham%20Goyal/AppData/Roaming/Python/Python310/site-packages/tensorflow/python/eager/function.py?line=503'>504</a>\u001b[0m         ctx\u001b[39m=\u001b[39;49mctx)\n\u001b[0;32m    <a href='file:///c%3A/Users/Saksham%20Goyal/AppData/Roaming/Python/Python310/site-packages/tensorflow/python/eager/function.py?line=504'>505</a>\u001b[0m   \u001b[39melse\u001b[39;00m:\n\u001b[0;32m    <a href='file:///c%3A/Users/Saksham%20Goyal/AppData/Roaming/Python/Python310/site-packages/tensorflow/python/eager/function.py?line=505'>506</a>\u001b[0m     outputs \u001b[39m=\u001b[39m execute\u001b[39m.\u001b[39mexecute_with_cancellation(\n\u001b[0;32m    <a href='file:///c%3A/Users/Saksham%20Goyal/AppData/Roaming/Python/Python310/site-packages/tensorflow/python/eager/function.py?line=506'>507</a>\u001b[0m         \u001b[39mstr\u001b[39m(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39msignature\u001b[39m.\u001b[39mname),\n\u001b[0;32m    <a href='file:///c%3A/Users/Saksham%20Goyal/AppData/Roaming/Python/Python310/site-packages/tensorflow/python/eager/function.py?line=507'>508</a>\u001b[0m         num_outputs\u001b[39m=\u001b[39m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_num_outputs,\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m    <a href='file:///c%3A/Users/Saksham%20Goyal/AppData/Roaming/Python/Python310/site-packages/tensorflow/python/eager/function.py?line=510'>511</a>\u001b[0m         ctx\u001b[39m=\u001b[39mctx,\n\u001b[0;32m    <a href='file:///c%3A/Users/Saksham%20Goyal/AppData/Roaming/Python/Python310/site-packages/tensorflow/python/eager/function.py?line=511'>512</a>\u001b[0m         cancellation_manager\u001b[39m=\u001b[39mcancellation_manager)\n",
      "File \u001b[1;32m~\\AppData\\Roaming\\Python\\Python310\\site-packages\\tensorflow\\python\\eager\\execute.py:54\u001b[0m, in \u001b[0;36mquick_execute\u001b[1;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[0;32m     <a href='file:///c%3A/Users/Saksham%20Goyal/AppData/Roaming/Python/Python310/site-packages/tensorflow/python/eager/execute.py?line=51'>52</a>\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m     <a href='file:///c%3A/Users/Saksham%20Goyal/AppData/Roaming/Python/Python310/site-packages/tensorflow/python/eager/execute.py?line=52'>53</a>\u001b[0m   ctx\u001b[39m.\u001b[39mensure_initialized()\n\u001b[1;32m---> <a href='file:///c%3A/Users/Saksham%20Goyal/AppData/Roaming/Python/Python310/site-packages/tensorflow/python/eager/execute.py?line=53'>54</a>\u001b[0m   tensors \u001b[39m=\u001b[39m pywrap_tfe\u001b[39m.\u001b[39;49mTFE_Py_Execute(ctx\u001b[39m.\u001b[39;49m_handle, device_name, op_name,\n\u001b[0;32m     <a href='file:///c%3A/Users/Saksham%20Goyal/AppData/Roaming/Python/Python310/site-packages/tensorflow/python/eager/execute.py?line=54'>55</a>\u001b[0m                                       inputs, attrs, num_outputs)\n\u001b[0;32m     <a href='file:///c%3A/Users/Saksham%20Goyal/AppData/Roaming/Python/Python310/site-packages/tensorflow/python/eager/execute.py?line=55'>56</a>\u001b[0m \u001b[39mexcept\u001b[39;00m core\u001b[39m.\u001b[39m_NotOkStatusException \u001b[39mas\u001b[39;00m e:\n\u001b[0;32m     <a href='file:///c%3A/Users/Saksham%20Goyal/AppData/Roaming/Python/Python310/site-packages/tensorflow/python/eager/execute.py?line=56'>57</a>\u001b[0m   \u001b[39mif\u001b[39;00m name \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# Load model from file\n",
    "# check the accuracy on the test set\n",
    "# show the summary of the model (i.e., the layers) as a graph\n",
    "pathtomodel = 'D:/Documents/Machine Learning/Project Files/best_model'\n",
    "model = base_cnn()\n",
    "model.load_weights(pathtomodel)\n",
    "import pydotplus as pydot\n",
    "from tensorflow.keras.utils import plot_model\n",
    "import keras.utils.vis_utils\n",
    "from tensorflow.keras import Model, Input\n",
    "from tensorflow.keras import layers\n",
    "tf.keras.utils.plot_model(model, to_file='model_summary.png', show_shapes=True, expand_nested=True)\n",
    "\n",
    "loss, acc = model.evaluate(ds_test, verbose=1)\n",
    "print(\"Restored model, accuracy: {:5.2f}%\".format(100 * acc))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Main GUI Application\n",
    "from keras.preprocessing.image import img_to_array\n",
    "image1 = PIL.Image.new(\"RGB\", (1000, 300), 'black')\n",
    "draw = ImageDraw.Draw(image1)\n",
    "app = Tk()\n",
    "app.geometry(\"1050x300\")\n",
    "app.resizable(0, 0)\n",
    "app.configure(background='grey')\n",
    "app.title(\"Multi-digit MNIST - Please draw the equation here\")\n",
    "lasx, lasy = 0, 0\n",
    "def get_x_and_y(event):\n",
    "\tglobal lasx, lasy\n",
    "\tlasx, lasy = event.x, event.y\n",
    "def draw_smth(event):\n",
    "\tglobal lasx, lasy, draw\n",
    "\tcanvas.create_line((lasx, lasy, event.x, event.y), width=7, fill='white', capstyle=ROUND)\n",
    "\tdraw.line((lasx, lasy, event.x, event.y), width=7, fill='white')\n",
    "\tlasx, lasy = event.x, event.y\n",
    "def clear_canv():\n",
    "\tglobal canvas\n",
    "\tapp.destroy()\n",
    "\tcanvas.delete('all')\n",
    "\tcanvas.create_line((700, 85,  800, 85),  width=7, fill='white', capstyle=ROUND)\n",
    "\tcanvas.create_line((700, 120, 800, 120), width=7, fill='white', capstyle=ROUND)\n",
    "\n",
    "def squareifyImage(roi):\n",
    "\told_height, old_width, channels = roi.shape\n",
    "\tif(old_height > old_width): \n",
    "\t\tnew_height, new_width = old_height, old_height\n",
    "\telse: \n",
    "\t\tnew_height, new_width = old_width, old_width\n",
    "\tresult = np.full((new_height,new_width, channels), (0,0,0), dtype=np.uint8)\n",
    "\tx_center = (new_width - old_width) // 2\n",
    "\ty_center = (new_height - old_height) // 2\n",
    "\tresult[y_center:y_center+old_height, x_center:x_center+old_width] = roi\n",
    "\treturn result\n",
    "\n",
    "def save_image():\n",
    "\timage1.save(f\"log.png\")\n",
    "\tim = cv2.imread(f\"log.png\")\n",
    "\tcontours, hierarchy = cv2.findContours(cv2.cvtColor(im,cv2.COLOR_BGR2GRAY),cv2.RETR_EXTERNAL,cv2.CHAIN_APPROX_SIMPLE)[-2:]\n",
    "\tsorted_ctrs = sorted(contours, key=lambda ctr: cv2.boundingRect(ctr)[0])\n",
    "\tlist_of_images = []\n",
    "\tcheck_images = []\n",
    "\tfor i, ctr in enumerate(sorted_ctrs):\n",
    "\t\t# Get bounding box\n",
    "\t\tx, y, w, h = cv2.boundingRect(ctr)\n",
    "\t\t# squareify image and resize to 28x28\n",
    "\t\tresult = squareifyImage(im[y:y+h, x:x+w])\n",
    "\t\timg_pad = cv2.copyMakeBorder(cv2.resize(result, (20, 20)), 4, 4, 4, 4, cv2.BORDER_CONSTANT, (0,0,0))\n",
    "\t\timg_pad = cv2.blur(img_pad, (2,2)) * 1.1\n",
    "\t\t# if image is on the left side, add to the left side of the list\n",
    "\t\t# if image is on the right side, add to the right side of the list\n",
    "\t\tif(x < 750):\n",
    "\t\t\tlist_of_images.append(img_to_array(img_pad[:,:,0]).reshape((28, 28, 1)))\n",
    "\t\telse:\n",
    "\t\t\tcheck_images.append(img_to_array(img_pad[:,:,0]).reshape((28, 28, 1)))\n",
    "\t\n",
    "\tresultstr = \"\"\n",
    "\tcheckstr = \"\"\n",
    "\tpreds2 = []\n",
    "\tpreds = model.predict(np.array(list_of_images))\n",
    "\tif(len(check_images) > 0):\n",
    "\t\tpreds2 = model.predict(np.array(check_images))\n",
    "\n",
    "\tfor pred in preds:\n",
    "\t\t#print(pred)\n",
    "\t\tif(class_names[np.argmax(pred)] == 4 and (pred[4] != 1.0)):\n",
    "\t\t\tresultstr += str(heapq.nlargest(2, xrange(len(pred)), key=pred.__getitem__)[1])\n",
    "\n",
    "\t\tif(np.argmax(pred) == 10):\n",
    "\t\t\tresultstr += \"+\"\n",
    "\t\telif (np.argmax(pred) == 11):\n",
    "\t\t\tresultstr += \"-\"\n",
    "\t\telif (np.argmax(pred) == 12):\n",
    "\t\t\tresultstr += \"*\"\n",
    "\t\telse:\n",
    "\t\t\tresultstr += str(class_names[np.argmax(pred)])\n",
    "\n",
    "\tfor pred in preds2:\n",
    "\t\t#print(pred)\n",
    "\t\tif(class_names[np.argmax(pred)] == 4 and (pred[4] != 1.0)):\n",
    "\t\t\t# if 4 is the largest class, and 4 is not the perfect guess, then the second largest class is the answer\n",
    "\t\t\t# sort the classes in descending order and get the index of the 2nd largest class\n",
    "\t\t\tcheckstr += str(heapq.nlargest(2, xrange(len(pred)), key=pred.__getitem__)[1])\n",
    "\n",
    "\t\tif(np.argmax(pred) == 10):\n",
    "\t\t\tcheckstr += \"+\"\n",
    "\t\telif (np.argmax(pred) == 11):\n",
    "\t\t\tcheckstr += \"-\"\n",
    "\t\telif (np.argmax(pred) == 12):\n",
    "\t\t\tcheckstr += \"*\"\n",
    "\t\telse:\n",
    "\t\t\tcheckstr += str(class_names[np.argmax(pred)])\n",
    "\n",
    "\n",
    "\tfor i in range(11):\n",
    "\t\ttry: os.remove(f'{i}.png')\n",
    "\t\texcept: pass\n",
    "\t\n",
    "\ttry:\n",
    "\t\tif(len(check_images) > 0):\n",
    "\t\t\tlabel.config(text=f\"{str(resultstr)+ '=' + str(checkstr)} => {eval(str(eval(resultstr))+ '==' + str(eval(checkstr)))}\")\n",
    "\t\telse:\n",
    "\t\t\tlabel.config(text=f\"{resultstr}={eval(resultstr)}\")\n",
    "\texcept:\n",
    "\t\tlabel.config(text=f\"Invalid Expression ({str(resultstr)+ '==' + str(checkstr)})\")\n",
    "\n",
    "canvas = Canvas(app, bg='black', width=1050, height=200)\n",
    "canvas.bind(\"<Button-1>\", get_x_and_y)\n",
    "canvas.bind(\"<B1-Motion>\", draw_smth)\n",
    "canvas.grid(row=0, column=0, pady=2, sticky=NSEW, columnspan=2)\n",
    "canvas.create_line((700, 85, 800, 85), width=7, fill='white', capstyle=ROUND)\n",
    "canvas.create_line((700, 120, 800, 120), width=7, fill='white', capstyle=ROUND)\n",
    "recognize = Button(master=app, text='Solve',width=15, height=2, command=save_image).grid(row=2, column=0, sticky=NSEW, pady=1, padx=1)\n",
    "clear_but = Button(master=app, text='Clear (not working yet)',width=15, height=2, command=clear_canv).grid(row=2, column=1, sticky=NSEW, pady=1, padx=1)\n",
    "label=Label(app, width=10, height=1, font=(\"Helvetica\", 30))\n",
    "label.grid(row=3, sticky=NSEW, pady=1, padx=1, columnspan=2)\n",
    "app.mainloop()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Do not run - normalize new dataset to match MNIST dataset parameters\n",
    "#impath  = 'D:/Documents/Machine Learning/Project Files/new datasets/mul/'\n",
    "#impath2 = 'D:/Documents/Machine Learning/Project Files/new datasets/N_mul/'\n",
    "\n",
    "\n",
    "#for i in range(10000):\n",
    "#\ttry:\n",
    "#\t\tavging = cv2.blur(cv2.imread(f'{impath}{str(i)}.png'),(10,10)) * 1.5\n",
    "#\t\tavging = cv2.resize(avging, (18, 18))\n",
    "#\t\tavging = cv2.copyMakeBorder(avging, 5, 5, 5, 5, cv2.BORDER_CONSTANT, (0,0,0))\n",
    "#\t\tcv2.imwrite(f'{impath2}{str(i)}.png', avging)\n",
    "#\t\t\n",
    "#\texcept:\n",
    "#\t\tpass"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "ML Project Testing",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}