Added image renaming file

Author: Frederik

.gitignore

@@ -107,8 +107,12 @@ venv.bak/
 
 # custom
 /data
+data/
+data__old/
 *.xml
 .idea/NIST.iml
 log/
 test/
-*.npy
+*.npy
+by_merge/
+models/

Preprocessing the data.md

@@ -0,0 +1,2 @@
+### 1. Downloading the data
+In this example, I will use the [Speical Database 19](https://www.nist.gov/srd/nist-special-database-19) published by the [National Institute for Standards And Technology](https://www.nist.gov/). It contains over 800,000 pre-classified images of handwritten letters and digits. It differentiates between 47 classes: All uppercase letters, all numbers and a few lower case letters. I downloaded the `by_merge.zip` file and saved in in my projects folder. To make working with the files easier, I wrote [a python script]() to move all the images into the same folder and rename them `class_[class]_index_[index].jpeg`, for example `class_25_Index_3743.jpeg`. 

models/32x32_3conv_16_32_64_1norm_512.ckpt.data-00000-of-00001

@@ -1,6 +1,6 @@
-model_checkpoint_path: "32x32_3conv_16_32_64_1norm_512.ckpt"
-all_model_checkpoint_paths: "..\\log\\model.ckpt-19400"
-all_model_checkpoint_paths: "..\\log\\model.ckpt-19600"
-all_model_checkpoint_paths: "..\\log\\model.ckpt-19800"
-all_model_checkpoint_paths: "..\\log\\model.ckpt-20000"
-all_model_checkpoint_paths: "32x32_3conv_16_32_64_1norm_512.ckpt"
+model_checkpoint_path: "32x32_2conv_32_64_1norm_1024.ckpt"
+all_model_checkpoint_paths: "..\\log\\model.ckpt-14400"
+all_model_checkpoint_paths: "..\\log\\model.ckpt-14600"
+all_model_checkpoint_paths: "..\\log\\model.ckpt-14800"
+all_model_checkpoint_paths: "..\\log\\model.ckpt-15000"
+all_model_checkpoint_paths: "32x32_2conv_32_64_1norm_1024.ckpt"

models/32x32_3conv_16_32_64_1norm_512.ckpt.index

@@ -79,7 +79,7 @@ def normal_full_layer(input_layer, size, act=tf.nn.relu, name="unspecified"):
 def main():
     single_image = dh.get_2d_array(sys.argv[1], (32,32,1))  # Convert image to numpy array
     X = np.array([single_image])  # Create array from image to fit shape of x (?,32,32,1)
-    checkpoint = "models/32x32_3conv_16_32_64_1norm_512.ckpt"  # Model used for prediction, must have the same graph structure!
+    checkpoint = "models/32x32_2conv_32_64_1norm_1024.ckpt"  # Model used for prediction, must have the same graph structure!
 
     # DICT
     classes = {
@@ -138,19 +138,20 @@ def main():
 
     # MODEL
     # filter size=(4,4); channels=1; filters=16; shape=?x32x32x32
-    convo_1 = convolutional_layer(x, shape=[4, 4, 1, 16], name="Convolutional_1")
+    convo_1 = convolutional_layer(x, shape=[4, 4, 1, 32], name="Convolutional_1")
     convo_1_pooling = max_pool_2by2(convo_1)  # shape=?x16x16x32
 
     # filter size=(4,4); channels=16; filters=32; shape=?x16x16x64
-    convo_2 = convolutional_layer(convo_1_pooling, shape=[4, 4, 16, 32], name="Convolutional_2")
+    convo_2 = convolutional_layer(convo_1_pooling, shape=[4, 4, 32, 64], name="Convolutional_2")
     convo_2_pooling = max_pool_2by2(convo_2)  # shape=?x8x8x64
+    convo_2_flat = tf.reshape(convo_2_pooling, [-1, 8 * 8 * 64])
 
     # filter size=(4,4); channels=32; filters=64; shape=?x8x8x32
-    convo_3 = convolutional_layer(convo_2_pooling, shape=[4, 4, 32, 64], name="Convolutional_3")
-    convo_3_pooling = max_pool_2by2(convo_3)  # shape=4x4x32
-    convo_3_flat = tf.reshape(convo_3_pooling, [-1, 4 * 4 * 64])  # Flatten convolutional layer
+    #convo_3 = convolutional_layer(convo_2_pooling, shape=[4, 4, 32, 64], name="Convolutional_3")
+    #convo_3_pooling = max_pool_2by2(convo_3)  # shape=4x4x32
+    #convo_3_flat = tf.reshape(convo_3_pooling, [-1, 4 * 4 * 64])  # Flatten convolutional layer
 
-    full_layer_one = normal_full_layer(convo_3_flat, 512, tf.nn.relu, name="Normal_Layer_1")
+    full_layer_one = normal_full_layer(convo_2_flat, 1024, tf.nn.relu, name="Normal_Layer_1")
     with tf.name_scope("dropout"):
         hold_prob = tf.placeholder(tf.float32)
         tf.summary.scalar("dropout_keep_probability", hold_prob)

models/32x32_3conv_16_32_64_1norm_512.ckpt.meta

@@ -0,0 +1,31 @@
+import os
+from shutil import copyfile
+
+by_merge_dir = "./by_merge"
+output_dir = "./data/"
+
+
+def get_class(str):
+    return str.split("\\")[1]
+
+
+def main():
+    index = 0
+    class_index = -1
+    classes = []
+    for subdir, dirs, files in os.walk(by_merge_dir):
+        for file in files:
+            if get_class(subdir) not in classes:
+                classes.append(get_class(subdir))
+                class_index += 1
+                index = 0
+            copyfile(os.path.join(subdir, file),
+                     os.path.join(output_dir, "class_" + str(class_index) + "_index_" + str(index) + ".png"))
+
+            print("Copied " + os.path.join(subdir, file) + " to "
+                  + os.path.join(output_dir, "class_" + str(class_index) + "_index_" + str(index) + ".png"))
+            index += 1
+
+
+if __name__ == "__main__":
+    main()

models/checkpoint

@@ -0,0 +1 @@
+tensorboard --logdir=D:/Coding/python/Tensorflow/NIST/log

predict.py

@@ -2,8 +2,11 @@
 import tensorflow as tf
 import math
 import sys
+import time
 import datetime
+import os
 
+os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
 
 class NISTHelper():
     def __init__(self, train_img, train_label, test_img, test_label):
@@ -132,19 +135,20 @@ def main():
 
     # MODEL
     # filter size=(4,4); channels=1; filters=16; shape=?x32x32x32
-    convo_1 = convolutional_layer(x, shape=[4, 4, 1, 16], name="Convolutional_1")
+    convo_1 = convolutional_layer(x, shape=[4, 4, 1, 32], name="Convolutional_1")
     convo_1_pooling = max_pool_2by2(convo_1)  # shape=?x16x16x32
 
     # filter size=(4,4); channels=16; filters=32; shape=?x16x16x64
-    convo_2 = convolutional_layer(convo_1_pooling, shape=[4, 4, 16, 32], name="Convolutional_2")
+    convo_2 = convolutional_layer(convo_1_pooling, shape=[4, 4, 32, 64], name="Convolutional_2")
     convo_2_pooling = max_pool_2by2(convo_2)  # shape=?x8x8x64
-
+    convo_2_flat = tf.reshape(convo_2_pooling, [-1, 8*8*64])
+    
     # filter size=(4,4); channels=32; filters=64; shape=?x8x8x32
-    convo_3 = convolutional_layer(convo_2_pooling, shape=[4, 4, 32, 64], name="Convolutional_3")
-    convo_3_pooling = max_pool_2by2(convo_3)  # shape=4x4x32
-    convo_3_flat = tf.reshape(convo_3_pooling, [-1, 4 * 4 * 64])  # Flatten convolutional layer
+    #convo_3 = convolutional_layer(convo_2_pooling, shape=[4, 4, 32, 64], name="Convolutional_3")
+    #convo_3_pooling = max_pool_2by2(convo_3)  # shape=4x4x32
+    #convo_3_flat = tf.reshape(convo_3_pooling, [-1, 4 * 4 * 64])  # Flatten convolutional layer
 
-    full_layer_one = normal_full_layer(convo_3_flat, 512, tf.nn.relu, name="Normal_Layer_1")
+    full_layer_one = normal_full_layer(convo_2_flat, 1024, tf.nn.relu, name="Normal_Layer_1")
     with tf.name_scope("dropout"):
         hold_prob = tf.placeholder(tf.float32)
         tf.summary.scalar("dropout_keep_probability", hold_prob)
@@ -154,11 +158,10 @@ def main():
     with tf.name_scope("cross_entropy"):
         with tf.name_scope("total"):
             # Calculate cross-entropy
-            cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y_true, logits=y_pred))
+            cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(labels=y_true, logits=y_pred))
     tf.summary.scalar("cross_entropy", cross_entropy)
     with tf.name_scope("train"):
-        optimizer = tf.train.AdamOptimizer(learning_rate=0.002)  # Optimizer
-        train = optimizer.minimize(cross_entropy)
+        train = tf.train.AdamOptimizer(learning_rate=0.001).minimize(cross_entropy)
     with tf.name_scope("accuracy"):
         with tf.name_scope("correct_predictions"):
             correct_predictions = tf.equal(tf.argmax(y_pred, 1), tf.argmax(y_true, 1))  # use argmax to get the index
@@ -172,7 +175,7 @@ def main():
     merged = tf.summary.merge_all()
     init = tf.global_variables_initializer()
     saver = tf.train.Saver()
-
+    epoch_start = 0
     with tf.Session() as sess:
         train_writer = tf.summary.FileWriter("log/train", sess.graph)
         test_writer = tf.summary.FileWriter("log/test")
@@ -193,9 +196,13 @@ def main():
                 summary, accuracy = sess.run([merged, acc], feed_dict={x: batch[0], y_true: batch[1], hold_prob: 1})
                 test_writer.add_summary(summary, i)  # Save the results of test batch
                 saver.save(sess, "log/model.ckpt", i)  # Save model
-                log("Step: " + str(i) + "; Accuracy: " + str(accuracy))
+                if i > 0:
+                    log("Step: " + str(i) + "; Accuracy: " + str(accuracy) + "; Time (200 Steps): " + str((time.time() - epoch_start)))
+                else:
+                    log("Step: " + str(i) + "; Accuracy: " + str(accuracy) + ";")
+                epoch_start = time.time()
         log("Finished training.")
-        model_path = "models/32x32_3conv_16_32_64_1norm_512.ckpt"
+        model_path = "models/32x32_2conv_32_64_1norm_1024.ckpt"
         saver.save(sess, model_path)  # Save final model
         log("Model saved in " + model_path)