Train_Model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/guest/.local/lib/python3.6/site-packages/cleverhans/utils_tf.py:345: The name tf.GraphKeys is deprecated. Please use tf.compat.v1.GraphKeys instead.\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "#!/usr/bin/henv python3\n",
    "# -*- coding: utf-8 -*-\n",
    "\n",
    "\"\"\"\n",
    "This script trains a model that uses ECOC coding. It defines many types of models (baseline and ensemble). \n",
    "Uncomment the final two lines corresponding to the model of interest from one of the below model definition \"code blocks\" to train that model. \n",
    "Next run \"AttackModel\" to then attack this model.\n",
    "\"\"\"\n",
    "\n",
    "#IMPORTS \n",
    "import os\n",
    "import tensorflow as tf\n",
    "tf.enable_eager_execution()\n",
    "tf.config.experimental_run_functions_eagerly(True)\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = '0' \n",
    "import numpy as np\n",
    "from tensorflow.keras.datasets import mnist, cifar10\n",
    "from Model_Implementations import Model_Softmax_Baseline, Model_Logistic_Baseline, Model_Logistic_Ensemble, Model_Tanh_Ensemble, Model_Tanh_Baseline\n",
    "import scipy.linalg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:\n",
      "The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
      "For more information, please see:\n",
      "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
      "  * https://github.com/tensorflow/addons\n",
      "  * https://github.com/tensorflow/io (for I/O related ops)\n",
      "If you depend on functionality not listed there, please file an issue.\n",
      "\n",
      "WARNING:tensorflow:From /home/guest/.local/lib/python3.6/site-packages/tensorflow_core/python/autograph/converters/directives.py:119: The name tf.random_uniform is deprecated. Please use tf.random.uniform instead.\n",
      "\n",
      "WARNING:tensorflow:From /home/guest/ours/DataAugmenter.py:41: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
      "WARNING:tensorflow:From /home/guest/.local/lib/python3.6/site-packages/tensorflow_core/python/autograph/converters/directives.py:119: The name tf.random_shuffle is deprecated. Please use tf.random.shuffle instead.\n",
      "\n",
      "Model: \"model\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input (InputLayer)              [(None, 32, 32, 3)]  0                                            \n",
      "__________________________________________________________________________________________________\n",
      "gaussian_noise (GaussianNoise)  (None, 32, 32, 3)    0           input[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "gaussian_noise_2 (GaussianNoise (None, 32, 32, 3)    0           input[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "data_augmenter (DataAugmenter)  (None, None, None, 3 0           gaussian_noise[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "data_augmenter_2 (DataAugmenter (None, None, None, 3 0           gaussian_noise_2[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "class_blender (ClassBlender)    (None, None, None, 3 0           data_augmenter[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "class_blender_2 (ClassBlender)  (None, None, None, 3 0           data_augmenter_2[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "clipper (Clipper)               (None, None, None, 3 0           class_blender[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "clipper_2 (Clipper)             (None, None, None, 3 0           class_blender_2[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "conv2d (Conv2D)                 (None, 32, 32, 32)   2432        clipper[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_34 (Conv2D)              (None, 32, 32, 32)   2432        clipper_2[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation (LWTA_ ((None, 32, 32, 32), 0           conv2d[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_34 (LW ((None, 32, 32, 32), 0           conv2d_34[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization (BatchNorma (None, 32, 32, 32)   128         lwta__conv2d__activation[0][0]   \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_8 (BatchNor (None, 32, 32, 32)   128         lwta__conv2d__activation_34[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "conv2d_1 (Conv2D)               (None, 32, 32, 32)   25632       batch_normalization[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_35 (Conv2D)              (None, 32, 32, 32)   25632       batch_normalization_8[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_1 (LWT ((None, 32, 32, 32), 0           conv2d_1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_35 (LW ((None, 32, 32, 32), 0           conv2d_35[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_1 (BatchNor (None, 32, 32, 32)   128         lwta__conv2d__activation_1[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_9 (BatchNor (None, 32, 32, 32)   128         lwta__conv2d__activation_35[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "conv2d_2 (Conv2D)               (None, 16, 16, 32)   9248        batch_normalization_1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_36 (Conv2D)              (None, 16, 16, 32)   9248        batch_normalization_9[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_2 (LWT ((None, 16, 16, 32), 0           conv2d_2[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_36 (LW ((None, 16, 16, 32), 0           conv2d_36[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_2 (BatchNor (None, 16, 16, 32)   128         lwta__conv2d__activation_2[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_10 (BatchNo (None, 16, 16, 32)   128         lwta__conv2d__activation_36[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "conv2d_3 (Conv2D)               (None, 16, 16, 64)   18496       batch_normalization_2[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_37 (Conv2D)              (None, 16, 16, 64)   18496       batch_normalization_10[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_3 (LWT ((None, 16, 16, 64), 0           conv2d_3[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_37 (LW ((None, 16, 16, 64), 0           conv2d_37[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_3 (BatchNor (None, 16, 16, 64)   256         lwta__conv2d__activation_3[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_11 (BatchNo (None, 16, 16, 64)   256         lwta__conv2d__activation_37[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "conv2d_4 (Conv2D)               (None, 16, 16, 64)   36928       batch_normalization_3[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_38 (Conv2D)              (None, 16, 16, 64)   36928       batch_normalization_11[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_4 (LWT ((None, 16, 16, 64), 0           conv2d_4[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_38 (LW ((None, 16, 16, 64), 0           conv2d_38[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_4 (BatchNor (None, 16, 16, 64)   256         lwta__conv2d__activation_4[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_12 (BatchNo (None, 16, 16, 64)   256         lwta__conv2d__activation_38[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "grayscaler (Grayscaler)         (None, 32, 32, 1)    0           input[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_5 (Conv2D)               (None, 8, 8, 64)     36928       batch_normalization_4[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "grayscaler_1 (Grayscaler)       (None, 32, 32, 1)    0           input[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_39 (Conv2D)              (None, 8, 8, 64)     36928       batch_normalization_12[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "gaussian_noise_1 (GaussianNoise (None, 32, 32, 1)    0           grayscaler[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_5 (LWT ((None, 8, 8, 64), ( 0           conv2d_5[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "gaussian_noise_3 (GaussianNoise (None, 32, 32, 1)    0           grayscaler_1[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_39 (LW ((None, 8, 8, 64), ( 0           conv2d_39[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "data_augmenter_1 (DataAugmenter (None, None, None, 1 0           gaussian_noise_1[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_5 (BatchNor (None, 8, 8, 64)     256         lwta__conv2d__activation_5[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "data_augmenter_3 (DataAugmenter (None, None, None, 1 0           gaussian_noise_3[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_13 (BatchNo (None, 8, 8, 64)     256         lwta__conv2d__activation_39[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "class_blender_1 (ClassBlender)  (None, None, None, 1 0           data_augmenter_1[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_6 (Conv2D)               (None, 8, 8, 128)    73856       batch_normalization_5[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "class_blender_3 (ClassBlender)  (None, None, None, 1 0           data_augmenter_3[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_40 (Conv2D)              (None, 8, 8, 128)    73856       batch_normalization_13[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "clipper_1 (Clipper)             (None, None, None, 1 0           class_blender_1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_6 (LWT ((None, 8, 8, 128),  0           conv2d_6[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "clipper_3 (Clipper)             (None, None, None, 1 0           class_blender_3[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_40 (LW ((None, 8, 8, 128),  0           conv2d_40[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_9 (Conv2D)               (None, 16, 16, 16)   416         clipper_1[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_6 (BatchNor (None, 8, 8, 128)    512         lwta__conv2d__activation_6[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_14 (Conv2D)              (None, 16, 16, 16)   416         clipper_1[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_19 (Conv2D)              (None, 16, 16, 16)   416         clipper_1[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_24 (Conv2D)              (None, 16, 16, 16)   416         clipper_1[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_29 (Conv2D)              (None, 16, 16, 16)   416         clipper_1[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_43 (Conv2D)              (None, 16, 16, 16)   416         clipper_3[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_14 (BatchNo (None, 8, 8, 128)    512         lwta__conv2d__activation_40[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "conv2d_48 (Conv2D)              (None, 16, 16, 16)   416         clipper_3[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_53 (Conv2D)              (None, 16, 16, 16)   416         clipper_3[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_58 (Conv2D)              (None, 16, 16, 16)   416         clipper_3[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_63 (Conv2D)              (None, 16, 16, 16)   416         clipper_3[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_9 (LWT ((None, 16, 16, 16), 0           conv2d_9[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_7 (Conv2D)               (None, 8, 8, 128)    147584      batch_normalization_6[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_14 (LW ((None, 16, 16, 16), 0           conv2d_14[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_19 (LW ((None, 16, 16, 16), 0           conv2d_19[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_24 (LW ((None, 16, 16, 16), 0           conv2d_24[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_29 (LW ((None, 16, 16, 16), 0           conv2d_29[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_43 (LW ((None, 16, 16, 16), 0           conv2d_43[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_41 (Conv2D)              (None, 8, 8, 128)    147584      batch_normalization_14[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_48 (LW ((None, 16, 16, 16), 0           conv2d_48[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_53 (LW ((None, 16, 16, 16), 0           conv2d_53[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_58 (LW ((None, 16, 16, 16), 0           conv2d_58[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_63 (LW ((None, 16, 16, 16), 0           conv2d_63[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_10 (Conv2D)              (None, 8, 8, 16)     2320        lwta__conv2d__activation_9[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_7 (LWT ((None, 8, 8, 128),  0           conv2d_7[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_15 (Conv2D)              (None, 8, 8, 16)     2320        lwta__conv2d__activation_14[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "conv2d_20 (Conv2D)              (None, 8, 8, 16)     2320        lwta__conv2d__activation_19[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "conv2d_25 (Conv2D)              (None, 8, 8, 16)     2320        lwta__conv2d__activation_24[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "conv2d_30 (Conv2D)              (None, 8, 8, 16)     2320        lwta__conv2d__activation_29[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "conv2d_44 (Conv2D)              (None, 8, 8, 16)     2320        lwta__conv2d__activation_43[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_41 (LW ((None, 8, 8, 128),  0           conv2d_41[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_49 (Conv2D)              (None, 8, 8, 16)     2320        lwta__conv2d__activation_48[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "conv2d_54 (Conv2D)              (None, 8, 8, 16)     2320        lwta__conv2d__activation_53[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "conv2d_59 (Conv2D)              (None, 8, 8, 16)     2320        lwta__conv2d__activation_58[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "conv2d_64 (Conv2D)              (None, 8, 8, 16)     2320        lwta__conv2d__activation_63[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_10 (LW ((None, 8, 8, 16), ( 0           conv2d_10[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_7 (BatchNor (None, 8, 8, 128)    512         lwta__conv2d__activation_7[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_15 (LW ((None, 8, 8, 16), ( 0           conv2d_15[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_20 (LW ((None, 8, 8, 16), ( 0           conv2d_20[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_25 (LW ((None, 8, 8, 16), ( 0           conv2d_25[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_30 (LW ((None, 8, 8, 16), ( 0           conv2d_30[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_44 (LW ((None, 8, 8, 16), ( 0           conv2d_44[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_15 (BatchNo (None, 8, 8, 128)    512         lwta__conv2d__activation_41[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_49 (LW ((None, 8, 8, 16), ( 0           conv2d_49[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_54 (LW ((None, 8, 8, 16), ( 0           conv2d_54[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_59 (LW ((None, 8, 8, 16), ( 0           conv2d_59[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_64 (LW ((None, 8, 8, 16), ( 0           conv2d_64[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_11 (Conv2D)              (None, 4, 4, 16)     2320        lwta__conv2d__activation_10[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "conv2d_8 (Conv2D)               (None, 4, 4, 128)    147584      batch_normalization_7[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_16 (Conv2D)              (None, 4, 4, 16)     2320        lwta__conv2d__activation_15[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "conv2d_21 (Conv2D)              (None, 4, 4, 16)     2320        lwta__conv2d__activation_20[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "conv2d_26 (Conv2D)              (None, 4, 4, 16)     2320        lwta__conv2d__activation_25[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "conv2d_31 (Conv2D)              (None, 4, 4, 16)     2320        lwta__conv2d__activation_30[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "conv2d_45 (Conv2D)              (None, 4, 4, 16)     2320        lwta__conv2d__activation_44[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "conv2d_42 (Conv2D)              (None, 4, 4, 128)    147584      batch_normalization_15[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_50 (Conv2D)              (None, 4, 4, 16)     2320        lwta__conv2d__activation_49[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "conv2d_55 (Conv2D)              (None, 4, 4, 16)     2320        lwta__conv2d__activation_54[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "conv2d_60 (Conv2D)              (None, 4, 4, 16)     2320        lwta__conv2d__activation_59[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "conv2d_65 (Conv2D)              (None, 4, 4, 16)     2320        lwta__conv2d__activation_64[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_11 (LW ((None, 4, 4, 16), ( 0           conv2d_11[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_8 (LWT ((None, 4, 4, 128),  0           conv2d_8[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_16 (LW ((None, 4, 4, 16), ( 0           conv2d_16[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_21 (LW ((None, 4, 4, 16), ( 0           conv2d_21[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_26 (LW ((None, 4, 4, 16), ( 0           conv2d_26[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_31 (LW ((None, 4, 4, 16), ( 0           conv2d_31[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_45 (LW ((None, 4, 4, 16), ( 0           conv2d_45[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_42 (LW ((None, 4, 4, 128),  0           conv2d_42[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_50 (LW ((None, 4, 4, 16), ( 0           conv2d_50[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_55 (LW ((None, 4, 4, 16), ( 0           conv2d_55[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_60 (LW ((None, 4, 4, 16), ( 0           conv2d_60[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_65 (LW ((None, 4, 4, 16), ( 0           conv2d_65[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate (Concatenate)       (None, 4, 4, 144)    0           lwta__conv2d__activation_11[0][0]\n",
      "                                                                 lwta__conv2d__activation_8[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1 (Concatenate)     (None, 4, 4, 144)    0           lwta__conv2d__activation_16[0][0]\n",
      "                                                                 lwta__conv2d__activation_8[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_2 (Concatenate)     (None, 4, 4, 144)    0           lwta__conv2d__activation_21[0][0]\n",
      "                                                                 lwta__conv2d__activation_8[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_3 (Concatenate)     (None, 4, 4, 144)    0           lwta__conv2d__activation_26[0][0]\n",
      "                                                                 lwta__conv2d__activation_8[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_4 (Concatenate)     (None, 4, 4, 144)    0           lwta__conv2d__activation_31[0][0]\n",
      "                                                                 lwta__conv2d__activation_8[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_7 (Concatenate)     (None, 4, 4, 144)    0           lwta__conv2d__activation_45[0][0]\n",
      "                                                                 lwta__conv2d__activation_42[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "concatenate_8 (Concatenate)     (None, 4, 4, 144)    0           lwta__conv2d__activation_50[0][0]\n",
      "                                                                 lwta__conv2d__activation_42[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "concatenate_9 (Concatenate)     (None, 4, 4, 144)    0           lwta__conv2d__activation_55[0][0]\n",
      "                                                                 lwta__conv2d__activation_42[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "concatenate_10 (Concatenate)    (None, 4, 4, 144)    0           lwta__conv2d__activation_60[0][0]\n",
      "                                                                 lwta__conv2d__activation_42[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "concatenate_11 (Concatenate)    (None, 4, 4, 144)    0           lwta__conv2d__activation_65[0][0]\n",
      "                                                                 lwta__conv2d__activation_42[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "conv2d_12 (Conv2D)              (None, 4, 4, 16)     9232        concatenate[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_17 (Conv2D)              (None, 4, 4, 16)     9232        concatenate_1[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_22 (Conv2D)              (None, 4, 4, 16)     9232        concatenate_2[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_27 (Conv2D)              (None, 4, 4, 16)     9232        concatenate_3[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_32 (Conv2D)              (None, 4, 4, 16)     9232        concatenate_4[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_46 (Conv2D)              (None, 4, 4, 16)     9232        concatenate_7[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_51 (Conv2D)              (None, 4, 4, 16)     9232        concatenate_8[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_56 (Conv2D)              (None, 4, 4, 16)     9232        concatenate_9[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_61 (Conv2D)              (None, 4, 4, 16)     9232        concatenate_10[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_66 (Conv2D)              (None, 4, 4, 16)     9232        concatenate_11[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_12 (LW ((None, 4, 4, 16), ( 0           conv2d_12[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_17 (LW ((None, 4, 4, 16), ( 0           conv2d_17[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_22 (LW ((None, 4, 4, 16), ( 0           conv2d_22[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_27 (LW ((None, 4, 4, 16), ( 0           conv2d_27[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_32 (LW ((None, 4, 4, 16), ( 0           conv2d_32[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_46 (LW ((None, 4, 4, 16), ( 0           conv2d_46[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_51 (LW ((None, 4, 4, 16), ( 0           conv2d_51[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_56 (LW ((None, 4, 4, 16), ( 0           conv2d_56[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_61 (LW ((None, 4, 4, 16), ( 0           conv2d_61[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_66 (LW ((None, 4, 4, 16), ( 0           conv2d_66[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_13 (Conv2D)              (None, 4, 4, 16)     1040        lwta__conv2d__activation_12[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "conv2d_18 (Conv2D)              (None, 4, 4, 16)     1040        lwta__conv2d__activation_17[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "conv2d_23 (Conv2D)              (None, 4, 4, 16)     1040        lwta__conv2d__activation_22[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "conv2d_28 (Conv2D)              (None, 4, 4, 16)     1040        lwta__conv2d__activation_27[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "conv2d_33 (Conv2D)              (None, 4, 4, 16)     1040        lwta__conv2d__activation_32[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "conv2d_47 (Conv2D)              (None, 4, 4, 16)     1040        lwta__conv2d__activation_46[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "conv2d_52 (Conv2D)              (None, 4, 4, 16)     1040        lwta__conv2d__activation_51[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "conv2d_57 (Conv2D)              (None, 4, 4, 16)     1040        lwta__conv2d__activation_56[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "conv2d_62 (Conv2D)              (None, 4, 4, 16)     1040        lwta__conv2d__activation_61[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "conv2d_67 (Conv2D)              (None, 4, 4, 16)     1040        lwta__conv2d__activation_66[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_13 (LW ((None, 4, 4, 16), ( 0           conv2d_13[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_18 (LW ((None, 4, 4, 16), ( 0           conv2d_18[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_23 (LW ((None, 4, 4, 16), ( 0           conv2d_23[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_28 (LW ((None, 4, 4, 16), ( 0           conv2d_28[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_33 (LW ((None, 4, 4, 16), ( 0           conv2d_33[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_47 (LW ((None, 4, 4, 16), ( 0           conv2d_47[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_52 (LW ((None, 4, 4, 16), ( 0           conv2d_52[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_57 (LW ((None, 4, 4, 16), ( 0           conv2d_57[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_62 (LW ((None, 4, 4, 16), ( 0           conv2d_62[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_67 (LW ((None, 4, 4, 16), ( 0           conv2d_67[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "flatten (Flatten)               (None, 256)          0           lwta__conv2d__activation_13[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "flatten_1 (Flatten)             (None, 256)          0           lwta__conv2d__activation_18[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "flatten_2 (Flatten)             (None, 256)          0           lwta__conv2d__activation_23[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "flatten_3 (Flatten)             (None, 256)          0           lwta__conv2d__activation_28[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "flatten_4 (Flatten)             (None, 256)          0           lwta__conv2d__activation_33[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "flatten_5 (Flatten)             (None, 256)          0           lwta__conv2d__activation_47[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "flatten_6 (Flatten)             (None, 256)          0           lwta__conv2d__activation_52[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "flatten_7 (Flatten)             (None, 256)          0           lwta__conv2d__activation_57[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "flatten_8 (Flatten)             (None, 256)          0           lwta__conv2d__activation_62[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "flatten_9 (Flatten)             (None, 256)          0           lwta__conv2d__activation_67[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "dense (Dense)                   (None, 16)           4112        flatten[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_4 (Dense)                 (None, 16)           4112        flatten_1[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_8 (Dense)                 (None, 16)           4112        flatten_2[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_12 (Dense)                (None, 16)           4112        flatten_3[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_16 (Dense)                (None, 16)           4112        flatten_4[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_20 (Dense)                (None, 16)           4112        flatten_5[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_24 (Dense)                (None, 16)           4112        flatten_6[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_28 (Dense)                (None, 16)           4112        flatten_7[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_32 (Dense)                (None, 16)           4112        flatten_8[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_36 (Dense)                (None, 16)           4112        flatten_9[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lwta__dense__activation (LWTA_D ((None, 16), (None,  0           dense[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "lwta__dense__activation_4 (LWTA ((None, 16), (None,  0           dense_4[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "lwta__dense__activation_8 (LWTA ((None, 16), (None,  0           dense_8[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "lwta__dense__activation_12 (LWT ((None, 16), (None,  0           dense_12[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lwta__dense__activation_16 (LWT ((None, 16), (None,  0           dense_16[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lwta__dense__activation_20 (LWT ((None, 16), (None,  0           dense_20[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lwta__dense__activation_24 (LWT ((None, 16), (None,  0           dense_24[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lwta__dense__activation_28 (LWT ((None, 16), (None,  0           dense_28[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lwta__dense__activation_32 (LWT ((None, 16), (None,  0           dense_32[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lwta__dense__activation_36 (LWT ((None, 16), (None,  0           dense_36[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 8)            136         lwta__dense__activation[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "dense_5 (Dense)                 (None, 8)            136         lwta__dense__activation_4[0][0]  \n",
      "__________________________________________________________________________________________________\n",
      "dense_9 (Dense)                 (None, 8)            136         lwta__dense__activation_8[0][0]  \n",
      "__________________________________________________________________________________________________\n",
      "dense_13 (Dense)                (None, 8)            136         lwta__dense__activation_12[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "dense_17 (Dense)                (None, 8)            136         lwta__dense__activation_16[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "dense_21 (Dense)                (None, 8)            136         lwta__dense__activation_20[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "dense_25 (Dense)                (None, 8)            136         lwta__dense__activation_24[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "dense_29 (Dense)                (None, 8)            136         lwta__dense__activation_28[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "dense_33 (Dense)                (None, 8)            136         lwta__dense__activation_32[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "dense_37 (Dense)                (None, 8)            136         lwta__dense__activation_36[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "lwta__dense__activation_1 (LWTA ((None, 8), (None, 4 0           dense_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "lwta__dense__activation_5 (LWTA ((None, 8), (None, 4 0           dense_5[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "lwta__dense__activation_9 (LWTA ((None, 8), (None, 4 0           dense_9[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "lwta__dense__activation_13 (LWT ((None, 8), (None, 4 0           dense_13[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lwta__dense__activation_17 (LWT ((None, 8), (None, 4 0           dense_17[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lwta__dense__activation_21 (LWT ((None, 8), (None, 4 0           dense_21[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lwta__dense__activation_25 (LWT ((None, 8), (None, 4 0           dense_25[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lwta__dense__activation_29 (LWT ((None, 8), (None, 4 0           dense_29[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lwta__dense__activation_33 (LWT ((None, 8), (None, 4 0           dense_33[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lwta__dense__activation_37 (LWT ((None, 8), (None, 4 0           dense_37[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_2 (Dense)                 (None, 4)            36          lwta__dense__activation_1[0][0]  \n",
      "__________________________________________________________________________________________________\n",
      "dense_6 (Dense)                 (None, 4)            36          lwta__dense__activation_5[0][0]  \n",
      "__________________________________________________________________________________________________\n",
      "dense_10 (Dense)                (None, 4)            36          lwta__dense__activation_9[0][0]  \n",
      "__________________________________________________________________________________________________\n",
      "dense_14 (Dense)                (None, 4)            36          lwta__dense__activation_13[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "dense_18 (Dense)                (None, 4)            36          lwta__dense__activation_17[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "dense_22 (Dense)                (None, 4)            36          lwta__dense__activation_21[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "dense_26 (Dense)                (None, 4)            36          lwta__dense__activation_25[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "dense_30 (Dense)                (None, 4)            36          lwta__dense__activation_29[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "dense_34 (Dense)                (None, 4)            36          lwta__dense__activation_33[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "dense_38 (Dense)                (None, 4)            36          lwta__dense__activation_37[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "lwta__dense__activation_2 (LWTA ((None, 4), (None, 2 0           dense_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "lwta__dense__activation_6 (LWTA ((None, 4), (None, 2 0           dense_6[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "lwta__dense__activation_10 (LWT ((None, 4), (None, 2 0           dense_10[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lwta__dense__activation_14 (LWT ((None, 4), (None, 2 0           dense_14[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lwta__dense__activation_18 (LWT ((None, 4), (None, 2 0           dense_18[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lwta__dense__activation_22 (LWT ((None, 4), (None, 2 0           dense_22[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lwta__dense__activation_26 (LWT ((None, 4), (None, 2 0           dense_26[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lwta__dense__activation_30 (LWT ((None, 4), (None, 2 0           dense_30[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lwta__dense__activation_34 (LWT ((None, 4), (None, 2 0           dense_34[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lwta__dense__activation_38 (LWT ((None, 4), (None, 2 0           dense_38[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_3 (Dense)                 (None, 2)            10          lwta__dense__activation_2[0][0]  \n",
      "__________________________________________________________________________________________________\n",
      "dense_7 (Dense)                 (None, 2)            10          lwta__dense__activation_6[0][0]  \n",
      "__________________________________________________________________________________________________\n",
      "dense_11 (Dense)                (None, 2)            10          lwta__dense__activation_10[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "dense_15 (Dense)                (None, 2)            10          lwta__dense__activation_14[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "dense_19 (Dense)                (None, 2)            10          lwta__dense__activation_18[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "dense_23 (Dense)                (None, 2)            10          lwta__dense__activation_22[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "dense_27 (Dense)                (None, 2)            10          lwta__dense__activation_26[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "dense_31 (Dense)                (None, 2)            10          lwta__dense__activation_30[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "dense_35 (Dense)                (None, 2)            10          lwta__dense__activation_34[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "dense_39 (Dense)                (None, 2)            10          lwta__dense__activation_38[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "lwta__dense__activation_3 (LWTA ((None, 2), (None, 1 0           dense_3[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "lwta__dense__activation_7 (LWTA ((None, 2), (None, 1 0           dense_7[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "lwta__dense__activation_11 (LWT ((None, 2), (None, 1 0           dense_11[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lwta__dense__activation_15 (LWT ((None, 2), (None, 1 0           dense_15[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lwta__dense__activation_19 (LWT ((None, 2), (None, 1 0           dense_19[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lwta__dense__activation_23 (LWT ((None, 2), (None, 1 0           dense_23[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lwta__dense__activation_27 (LWT ((None, 2), (None, 1 0           dense_27[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lwta__dense__activation_31 (LWT ((None, 2), (None, 1 0           dense_31[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lwta__dense__activation_35 (LWT ((None, 2), (None, 1 0           dense_35[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lwta__dense__activation_39 (LWT ((None, 2), (None, 1 0           dense_39[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "sb__layer (SB_Layer)            (None, 1)            9           lwta__dense__activation_3[0][0]  \n",
      "__________________________________________________________________________________________________\n",
      "sb__layer_1 (SB_Layer)          (None, 1)            9           lwta__dense__activation_7[0][0]  \n",
      "__________________________________________________________________________________________________\n",
      "sb__layer_2 (SB_Layer)          (None, 1)            9           lwta__dense__activation_11[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "sb__layer_3 (SB_Layer)          (None, 1)            9           lwta__dense__activation_15[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "sb__layer_4 (SB_Layer)          (None, 1)            9           lwta__dense__activation_19[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "sb__layer_5 (SB_Layer)          (None, 1)            9           lwta__dense__activation_23[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "sb__layer_6 (SB_Layer)          (None, 1)            9           lwta__dense__activation_27[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "sb__layer_7 (SB_Layer)          (None, 1)            9           lwta__dense__activation_31[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "sb__layer_8 (SB_Layer)          (None, 1)            9           lwta__dense__activation_35[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "sb__layer_9 (SB_Layer)          (None, 1)            9           lwta__dense__activation_39[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_6 (Concatenate)     (None, 5)            0           sb__layer[0][0]                  \n",
      "                                                                 sb__layer_1[0][0]                \n",
      "                                                                 sb__layer_2[0][0]                \n",
      "                                                                 sb__layer_3[0][0]                \n",
      "                                                                 sb__layer_4[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_13 (Concatenate)    (None, 5)            0           sb__layer_5[0][0]                \n",
      "                                                                 sb__layer_6[0][0]                \n",
      "                                                                 sb__layer_7[0][0]                \n",
      "                                                                 sb__layer_8[0][0]                \n",
      "                                                                 sb__layer_9[0][0]                \n",
      "==================================================================================================\n",
      "Total params: 1,198,038\n",
      "Trainable params: 1,195,862\n",
      "Non-trainable params: 2,176\n",
      "__________________________________________________________________________________________________\n",
      "None\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_1\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input (InputLayer)              [(None, 32, 32, 3)]  0                                            \n",
      "__________________________________________________________________________________________________\n",
      "gaussian_noise_4 (GaussianNoise (None, 32, 32, 3)    0           input[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "gaussian_noise_6 (GaussianNoise (None, 32, 32, 3)    0           input[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "data_augmenter_4 (DataAugmenter (None, None, None, 3 0           gaussian_noise_4[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "data_augmenter_6 (DataAugmenter (None, None, None, 3 0           gaussian_noise_6[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "class_blender_4 (ClassBlender)  (None, None, None, 3 0           data_augmenter_4[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "class_blender_6 (ClassBlender)  (None, None, None, 3 0           data_augmenter_6[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "clipper_4 (Clipper)             (None, None, None, 3 0           class_blender_4[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "clipper_6 (Clipper)             (None, None, None, 3 0           class_blender_6[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_68 (Conv2D)              (None, 32, 32, 32)   2432        clipper_4[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_102 (Conv2D)             (None, 32, 32, 32)   2432        clipper_6[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_68 (LW ((None, 32, 32, 32), 0           conv2d_68[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_102 (L ((None, 32, 32, 32), 0           conv2d_102[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_16 (BatchNo (None, 32, 32, 32)   128         lwta__conv2d__activation_68[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_24 (BatchNo (None, 32, 32, 32)   128         lwta__conv2d__activation_102[0][0\n",
      "__________________________________________________________________________________________________\n",
      "conv2d_69 (Conv2D)              (None, 32, 32, 32)   25632       batch_normalization_16[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_103 (Conv2D)             (None, 32, 32, 32)   25632       batch_normalization_24[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_69 (LW ((None, 32, 32, 32), 0           conv2d_69[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_103 (L ((None, 32, 32, 32), 0           conv2d_103[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_17 (BatchNo (None, 32, 32, 32)   128         lwta__conv2d__activation_69[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_25 (BatchNo (None, 32, 32, 32)   128         lwta__conv2d__activation_103[0][0\n",
      "__________________________________________________________________________________________________\n",
      "conv2d_70 (Conv2D)              (None, 16, 16, 32)   9248        batch_normalization_17[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_104 (Conv2D)             (None, 16, 16, 32)   9248        batch_normalization_25[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_70 (LW ((None, 16, 16, 32), 0           conv2d_70[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_104 (L ((None, 16, 16, 32), 0           conv2d_104[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_18 (BatchNo (None, 16, 16, 32)   128         lwta__conv2d__activation_70[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_26 (BatchNo (None, 16, 16, 32)   128         lwta__conv2d__activation_104[0][0\n",
      "__________________________________________________________________________________________________\n",
      "conv2d_71 (Conv2D)              (None, 16, 16, 64)   18496       batch_normalization_18[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_105 (Conv2D)             (None, 16, 16, 64)   18496       batch_normalization_26[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_71 (LW ((None, 16, 16, 64), 0           conv2d_71[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_105 (L ((None, 16, 16, 64), 0           conv2d_105[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_19 (BatchNo (None, 16, 16, 64)   256         lwta__conv2d__activation_71[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_27 (BatchNo (None, 16, 16, 64)   256         lwta__conv2d__activation_105[0][0\n",
      "__________________________________________________________________________________________________\n",
      "conv2d_72 (Conv2D)              (None, 16, 16, 64)   36928       batch_normalization_19[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_106 (Conv2D)             (None, 16, 16, 64)   36928       batch_normalization_27[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_72 (LW ((None, 16, 16, 64), 0           conv2d_72[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_106 (L ((None, 16, 16, 64), 0           conv2d_106[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_20 (BatchNo (None, 16, 16, 64)   256         lwta__conv2d__activation_72[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_28 (BatchNo (None, 16, 16, 64)   256         lwta__conv2d__activation_106[0][0\n",
      "__________________________________________________________________________________________________\n",
      "grayscaler_2 (Grayscaler)       (None, 32, 32, 1)    0           input[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_73 (Conv2D)              (None, 8, 8, 64)     36928       batch_normalization_20[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "grayscaler_3 (Grayscaler)       (None, 32, 32, 1)    0           input[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_107 (Conv2D)             (None, 8, 8, 64)     36928       batch_normalization_28[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "gaussian_noise_5 (GaussianNoise (None, 32, 32, 1)    0           grayscaler_2[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_73 (LW ((None, 8, 8, 64), ( 0           conv2d_73[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "gaussian_noise_7 (GaussianNoise (None, 32, 32, 1)    0           grayscaler_3[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_107 (L ((None, 8, 8, 64), ( 0           conv2d_107[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "data_augmenter_5 (DataAugmenter (None, None, None, 1 0           gaussian_noise_5[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_21 (BatchNo (None, 8, 8, 64)     256         lwta__conv2d__activation_73[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "data_augmenter_7 (DataAugmenter (None, None, None, 1 0           gaussian_noise_7[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_29 (BatchNo (None, 8, 8, 64)     256         lwta__conv2d__activation_107[0][0\n",
      "__________________________________________________________________________________________________\n",
      "class_blender_5 (ClassBlender)  (None, None, None, 1 0           data_augmenter_5[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_74 (Conv2D)              (None, 8, 8, 128)    73856       batch_normalization_21[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "class_blender_7 (ClassBlender)  (None, None, None, 1 0           data_augmenter_7[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_108 (Conv2D)             (None, 8, 8, 128)    73856       batch_normalization_29[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "clipper_5 (Clipper)             (None, None, None, 1 0           class_blender_5[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_74 (LW ((None, 8, 8, 128),  0           conv2d_74[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "clipper_7 (Clipper)             (None, None, None, 1 0           class_blender_7[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_108 (L ((None, 8, 8, 128),  0           conv2d_108[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_77 (Conv2D)              (None, 16, 16, 16)   416         clipper_5[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_22 (BatchNo (None, 8, 8, 128)    512         lwta__conv2d__activation_74[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "conv2d_82 (Conv2D)              (None, 16, 16, 16)   416         clipper_5[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_87 (Conv2D)              (None, 16, 16, 16)   416         clipper_5[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_92 (Conv2D)              (None, 16, 16, 16)   416         clipper_5[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_97 (Conv2D)              (None, 16, 16, 16)   416         clipper_5[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_111 (Conv2D)             (None, 16, 16, 16)   416         clipper_7[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_30 (BatchNo (None, 8, 8, 128)    512         lwta__conv2d__activation_108[0][0\n",
      "__________________________________________________________________________________________________\n",
      "conv2d_116 (Conv2D)             (None, 16, 16, 16)   416         clipper_7[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_121 (Conv2D)             (None, 16, 16, 16)   416         clipper_7[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_126 (Conv2D)             (None, 16, 16, 16)   416         clipper_7[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_131 (Conv2D)             (None, 16, 16, 16)   416         clipper_7[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_77 (LW ((None, 16, 16, 16), 0           conv2d_77[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_75 (Conv2D)              (None, 8, 8, 128)    147584      batch_normalization_22[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_82 (LW ((None, 16, 16, 16), 0           conv2d_82[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_87 (LW ((None, 16, 16, 16), 0           conv2d_87[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_92 (LW ((None, 16, 16, 16), 0           conv2d_92[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_97 (LW ((None, 16, 16, 16), 0           conv2d_97[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_111 (L ((None, 16, 16, 16), 0           conv2d_111[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_109 (Conv2D)             (None, 8, 8, 128)    147584      batch_normalization_30[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_116 (L ((None, 16, 16, 16), 0           conv2d_116[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_121 (L ((None, 16, 16, 16), 0           conv2d_121[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_126 (L ((None, 16, 16, 16), 0           conv2d_126[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_131 (L ((None, 16, 16, 16), 0           conv2d_131[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_78 (Conv2D)              (None, 8, 8, 16)     2320        lwta__conv2d__activation_77[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_75 (LW ((None, 8, 8, 128),  0           conv2d_75[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_83 (Conv2D)              (None, 8, 8, 16)     2320        lwta__conv2d__activation_82[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "conv2d_88 (Conv2D)              (None, 8, 8, 16)     2320        lwta__conv2d__activation_87[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "conv2d_93 (Conv2D)              (None, 8, 8, 16)     2320        lwta__conv2d__activation_92[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "conv2d_98 (Conv2D)              (None, 8, 8, 16)     2320        lwta__conv2d__activation_97[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "conv2d_112 (Conv2D)             (None, 8, 8, 16)     2320        lwta__conv2d__activation_111[0][0\n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_109 (L ((None, 8, 8, 128),  0           conv2d_109[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_117 (Conv2D)             (None, 8, 8, 16)     2320        lwta__conv2d__activation_116[0][0\n",
      "__________________________________________________________________________________________________\n",
      "conv2d_122 (Conv2D)             (None, 8, 8, 16)     2320        lwta__conv2d__activation_121[0][0\n",
      "__________________________________________________________________________________________________\n",
      "conv2d_127 (Conv2D)             (None, 8, 8, 16)     2320        lwta__conv2d__activation_126[0][0\n",
      "__________________________________________________________________________________________________\n",
      "conv2d_132 (Conv2D)             (None, 8, 8, 16)     2320        lwta__conv2d__activation_131[0][0\n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_78 (LW ((None, 8, 8, 16), ( 0           conv2d_78[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_23 (BatchNo (None, 8, 8, 128)    512         lwta__conv2d__activation_75[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_83 (LW ((None, 8, 8, 16), ( 0           conv2d_83[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_88 (LW ((None, 8, 8, 16), ( 0           conv2d_88[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_93 (LW ((None, 8, 8, 16), ( 0           conv2d_93[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_98 (LW ((None, 8, 8, 16), ( 0           conv2d_98[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_112 (L ((None, 8, 8, 16), ( 0           conv2d_112[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_31 (BatchNo (None, 8, 8, 128)    512         lwta__conv2d__activation_109[0][0\n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_117 (L ((None, 8, 8, 16), ( 0           conv2d_117[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_122 (L ((None, 8, 8, 16), ( 0           conv2d_122[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_127 (L ((None, 8, 8, 16), ( 0           conv2d_127[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_132 (L ((None, 8, 8, 16), ( 0           conv2d_132[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_79 (Conv2D)              (None, 4, 4, 16)     2320        lwta__conv2d__activation_78[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "conv2d_76 (Conv2D)              (None, 4, 4, 128)    147584      batch_normalization_23[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_84 (Conv2D)              (None, 4, 4, 16)     2320        lwta__conv2d__activation_83[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "conv2d_89 (Conv2D)              (None, 4, 4, 16)     2320        lwta__conv2d__activation_88[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "conv2d_94 (Conv2D)              (None, 4, 4, 16)     2320        lwta__conv2d__activation_93[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "conv2d_99 (Conv2D)              (None, 4, 4, 16)     2320        lwta__conv2d__activation_98[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "conv2d_113 (Conv2D)             (None, 4, 4, 16)     2320        lwta__conv2d__activation_112[0][0\n",
      "__________________________________________________________________________________________________\n",
      "conv2d_110 (Conv2D)             (None, 4, 4, 128)    147584      batch_normalization_31[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_118 (Conv2D)             (None, 4, 4, 16)     2320        lwta__conv2d__activation_117[0][0\n",
      "__________________________________________________________________________________________________\n",
      "conv2d_123 (Conv2D)             (None, 4, 4, 16)     2320        lwta__conv2d__activation_122[0][0\n",
      "__________________________________________________________________________________________________\n",
      "conv2d_128 (Conv2D)             (None, 4, 4, 16)     2320        lwta__conv2d__activation_127[0][0\n",
      "__________________________________________________________________________________________________\n",
      "conv2d_133 (Conv2D)             (None, 4, 4, 16)     2320        lwta__conv2d__activation_132[0][0\n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_79 (LW ((None, 4, 4, 16), ( 0           conv2d_79[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_76 (LW ((None, 4, 4, 128),  0           conv2d_76[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_84 (LW ((None, 4, 4, 16), ( 0           conv2d_84[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_89 (LW ((None, 4, 4, 16), ( 0           conv2d_89[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_94 (LW ((None, 4, 4, 16), ( 0           conv2d_94[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_99 (LW ((None, 4, 4, 16), ( 0           conv2d_99[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_113 (L ((None, 4, 4, 16), ( 0           conv2d_113[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_110 (L ((None, 4, 4, 128),  0           conv2d_110[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_118 (L ((None, 4, 4, 16), ( 0           conv2d_118[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_123 (L ((None, 4, 4, 16), ( 0           conv2d_123[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_128 (L ((None, 4, 4, 16), ( 0           conv2d_128[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_133 (L ((None, 4, 4, 16), ( 0           conv2d_133[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_14 (Concatenate)    (None, 4, 4, 144)    0           lwta__conv2d__activation_79[0][0]\n",
      "                                                                 lwta__conv2d__activation_76[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "concatenate_15 (Concatenate)    (None, 4, 4, 144)    0           lwta__conv2d__activation_84[0][0]\n",
      "                                                                 lwta__conv2d__activation_76[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "concatenate_16 (Concatenate)    (None, 4, 4, 144)    0           lwta__conv2d__activation_89[0][0]\n",
      "                                                                 lwta__conv2d__activation_76[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "concatenate_17 (Concatenate)    (None, 4, 4, 144)    0           lwta__conv2d__activation_94[0][0]\n",
      "                                                                 lwta__conv2d__activation_76[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "concatenate_18 (Concatenate)    (None, 4, 4, 144)    0           lwta__conv2d__activation_99[0][0]\n",
      "                                                                 lwta__conv2d__activation_76[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "concatenate_21 (Concatenate)    (None, 4, 4, 144)    0           lwta__conv2d__activation_113[0][0\n",
      "                                                                 lwta__conv2d__activation_110[0][0\n",
      "__________________________________________________________________________________________________\n",
      "concatenate_22 (Concatenate)    (None, 4, 4, 144)    0           lwta__conv2d__activation_118[0][0\n",
      "                                                                 lwta__conv2d__activation_110[0][0\n",
      "__________________________________________________________________________________________________\n",
      "concatenate_23 (Concatenate)    (None, 4, 4, 144)    0           lwta__conv2d__activation_123[0][0\n",
      "                                                                 lwta__conv2d__activation_110[0][0\n",
      "__________________________________________________________________________________________________\n",
      "concatenate_24 (Concatenate)    (None, 4, 4, 144)    0           lwta__conv2d__activation_128[0][0\n",
      "                                                                 lwta__conv2d__activation_110[0][0\n",
      "__________________________________________________________________________________________________\n",
      "concatenate_25 (Concatenate)    (None, 4, 4, 144)    0           lwta__conv2d__activation_133[0][0\n",
      "                                                                 lwta__conv2d__activation_110[0][0\n",
      "__________________________________________________________________________________________________\n",
      "conv2d_80 (Conv2D)              (None, 4, 4, 16)     9232        concatenate_14[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_85 (Conv2D)              (None, 4, 4, 16)     9232        concatenate_15[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_90 (Conv2D)              (None, 4, 4, 16)     9232        concatenate_16[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_95 (Conv2D)              (None, 4, 4, 16)     9232        concatenate_17[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_100 (Conv2D)             (None, 4, 4, 16)     9232        concatenate_18[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_114 (Conv2D)             (None, 4, 4, 16)     9232        concatenate_21[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_119 (Conv2D)             (None, 4, 4, 16)     9232        concatenate_22[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_124 (Conv2D)             (None, 4, 4, 16)     9232        concatenate_23[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_129 (Conv2D)             (None, 4, 4, 16)     9232        concatenate_24[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_134 (Conv2D)             (None, 4, 4, 16)     9232        concatenate_25[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_80 (LW ((None, 4, 4, 16), ( 0           conv2d_80[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_85 (LW ((None, 4, 4, 16), ( 0           conv2d_85[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_90 (LW ((None, 4, 4, 16), ( 0           conv2d_90[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_95 (LW ((None, 4, 4, 16), ( 0           conv2d_95[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_100 (L ((None, 4, 4, 16), ( 0           conv2d_100[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_114 (L ((None, 4, 4, 16), ( 0           conv2d_114[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_119 (L ((None, 4, 4, 16), ( 0           conv2d_119[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_124 (L ((None, 4, 4, 16), ( 0           conv2d_124[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_129 (L ((None, 4, 4, 16), ( 0           conv2d_129[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_134 (L ((None, 4, 4, 16), ( 0           conv2d_134[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_81 (Conv2D)              (None, 4, 4, 16)     1040        lwta__conv2d__activation_80[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "conv2d_86 (Conv2D)              (None, 4, 4, 16)     1040        lwta__conv2d__activation_85[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "conv2d_91 (Conv2D)              (None, 4, 4, 16)     1040        lwta__conv2d__activation_90[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "conv2d_96 (Conv2D)              (None, 4, 4, 16)     1040        lwta__conv2d__activation_95[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "conv2d_101 (Conv2D)             (None, 4, 4, 16)     1040        lwta__conv2d__activation_100[0][0\n",
      "__________________________________________________________________________________________________\n",
      "conv2d_115 (Conv2D)             (None, 4, 4, 16)     1040        lwta__conv2d__activation_114[0][0\n",
      "__________________________________________________________________________________________________\n",
      "conv2d_120 (Conv2D)             (None, 4, 4, 16)     1040        lwta__conv2d__activation_119[0][0\n",
      "__________________________________________________________________________________________________\n",
      "conv2d_125 (Conv2D)             (None, 4, 4, 16)     1040        lwta__conv2d__activation_124[0][0\n",
      "__________________________________________________________________________________________________\n",
      "conv2d_130 (Conv2D)             (None, 4, 4, 16)     1040        lwta__conv2d__activation_129[0][0\n",
      "__________________________________________________________________________________________________\n",
      "conv2d_135 (Conv2D)             (None, 4, 4, 16)     1040        lwta__conv2d__activation_134[0][0\n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_81 (LW ((None, 4, 4, 16), ( 0           conv2d_81[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_86 (LW ((None, 4, 4, 16), ( 0           conv2d_86[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_91 (LW ((None, 4, 4, 16), ( 0           conv2d_91[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_96 (LW ((None, 4, 4, 16), ( 0           conv2d_96[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_101 (L ((None, 4, 4, 16), ( 0           conv2d_101[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_115 (L ((None, 4, 4, 16), ( 0           conv2d_115[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_120 (L ((None, 4, 4, 16), ( 0           conv2d_120[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_125 (L ((None, 4, 4, 16), ( 0           conv2d_125[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_130 (L ((None, 4, 4, 16), ( 0           conv2d_130[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "lwta__conv2d__activation_135 (L ((None, 4, 4, 16), ( 0           conv2d_135[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "flatten_10 (Flatten)            (None, 256)          0           lwta__conv2d__activation_81[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "flatten_11 (Flatten)            (None, 256)          0           lwta__conv2d__activation_86[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "flatten_12 (Flatten)            (None, 256)          0           lwta__conv2d__activation_91[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "flatten_13 (Flatten)            (None, 256)          0           lwta__conv2d__activation_96[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "flatten_14 (Flatten)            (None, 256)          0           lwta__conv2d__activation_101[0][0\n",
      "__________________________________________________________________________________________________\n",
      "flatten_15 (Flatten)            (None, 256)          0           lwta__conv2d__activation_115[0][0\n",
      "__________________________________________________________________________________________________\n",
      "flatten_16 (Flatten)            (None, 256)          0           lwta__conv2d__activation_120[0][0\n",
      "__________________________________________________________________________________________________\n",
      "flatten_17 (Flatten)            (None, 256)          0           lwta__conv2d__activation_125[0][0\n",
      "__________________________________________________________________________________________________\n",
      "flatten_18 (Flatten)            (None, 256)          0           lwta__conv2d__activation_130[0][0\n",
      "__________________________________________________________________________________________________\n",
      "flatten_19 (Flatten)            (None, 256)          0           lwta__conv2d__activation_135[0][0\n",
      "__________________________________________________________________________________________________\n",
      "dense_40 (Dense)                (None, 16)           4112        flatten_10[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dense_44 (Dense)                (None, 16)           4112        flatten_11[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dense_48 (Dense)                (None, 16)           4112        flatten_12[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dense_52 (Dense)                (None, 16)           4112        flatten_13[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dense_56 (Dense)                (None, 16)           4112        flatten_14[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dense_60 (Dense)                (None, 16)           4112        flatten_15[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dense_64 (Dense)                (None, 16)           4112        flatten_16[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dense_68 (Dense)                (None, 16)           4112        flatten_17[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dense_72 (Dense)                (None, 16)           4112        flatten_18[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dense_76 (Dense)                (None, 16)           4112        flatten_19[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "lwta__dense__activation_40 (LWT ((None, 16), (None,  0           dense_40[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lwta__dense__activation_44 (LWT ((None, 16), (None,  0           dense_44[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lwta__dense__activation_48 (LWT ((None, 16), (None,  0           dense_48[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lwta__dense__activation_52 (LWT ((None, 16), (None,  0           dense_52[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lwta__dense__activation_56 (LWT ((None, 16), (None,  0           dense_56[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lwta__dense__activation_60 (LWT ((None, 16), (None,  0           dense_60[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lwta__dense__activation_64 (LWT ((None, 16), (None,  0           dense_64[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lwta__dense__activation_68 (LWT ((None, 16), (None,  0           dense_68[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lwta__dense__activation_72 (LWT ((None, 16), (None,  0           dense_72[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lwta__dense__activation_76 (LWT ((None, 16), (None,  0           dense_76[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_41 (Dense)                (None, 8)            136         lwta__dense__activation_40[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "dense_45 (Dense)                (None, 8)            136         lwta__dense__activation_44[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "dense_49 (Dense)                (None, 8)            136         lwta__dense__activation_48[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "dense_53 (Dense)                (None, 8)            136         lwta__dense__activation_52[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "dense_57 (Dense)                (None, 8)            136         lwta__dense__activation_56[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "dense_61 (Dense)                (None, 8)            136         lwta__dense__activation_60[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "dense_65 (Dense)                (None, 8)            136         lwta__dense__activation_64[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "dense_69 (Dense)                (None, 8)            136         lwta__dense__activation_68[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "dense_73 (Dense)                (None, 8)            136         lwta__dense__activation_72[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "dense_77 (Dense)                (None, 8)            136         lwta__dense__activation_76[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "lwta__dense__activation_41 (LWT ((None, 8), (None, 4 0           dense_41[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lwta__dense__activation_45 (LWT ((None, 8), (None, 4 0           dense_45[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lwta__dense__activation_49 (LWT ((None, 8), (None, 4 0           dense_49[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lwta__dense__activation_53 (LWT ((None, 8), (None, 4 0           dense_53[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lwta__dense__activation_57 (LWT ((None, 8), (None, 4 0           dense_57[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lwta__dense__activation_61 (LWT ((None, 8), (None, 4 0           dense_61[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lwta__dense__activation_65 (LWT ((None, 8), (None, 4 0           dense_65[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lwta__dense__activation_69 (LWT ((None, 8), (None, 4 0           dense_69[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lwta__dense__activation_73 (LWT ((None, 8), (None, 4 0           dense_73[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lwta__dense__activation_77 (LWT ((None, 8), (None, 4 0           dense_77[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_42 (Dense)                (None, 4)            36          lwta__dense__activation_41[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "dense_46 (Dense)                (None, 4)            36          lwta__dense__activation_45[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "dense_50 (Dense)                (None, 4)            36          lwta__dense__activation_49[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "dense_54 (Dense)                (None, 4)            36          lwta__dense__activation_53[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "dense_58 (Dense)                (None, 4)            36          lwta__dense__activation_57[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "dense_62 (Dense)                (None, 4)            36          lwta__dense__activation_61[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "dense_66 (Dense)                (None, 4)            36          lwta__dense__activation_65[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "dense_70 (Dense)                (None, 4)            36          lwta__dense__activation_69[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "dense_74 (Dense)                (None, 4)            36          lwta__dense__activation_73[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "dense_78 (Dense)                (None, 4)            36          lwta__dense__activation_77[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "lwta__dense__activation_42 (LWT ((None, 4), (None, 2 0           dense_42[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lwta__dense__activation_46 (LWT ((None, 4), (None, 2 0           dense_46[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lwta__dense__activation_50 (LWT ((None, 4), (None, 2 0           dense_50[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lwta__dense__activation_54 (LWT ((None, 4), (None, 2 0           dense_54[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lwta__dense__activation_58 (LWT ((None, 4), (None, 2 0           dense_58[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lwta__dense__activation_62 (LWT ((None, 4), (None, 2 0           dense_62[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lwta__dense__activation_66 (LWT ((None, 4), (None, 2 0           dense_66[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lwta__dense__activation_70 (LWT ((None, 4), (None, 2 0           dense_70[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lwta__dense__activation_74 (LWT ((None, 4), (None, 2 0           dense_74[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lwta__dense__activation_78 (LWT ((None, 4), (None, 2 0           dense_78[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_43 (Dense)                (None, 2)            10          lwta__dense__activation_42[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "dense_47 (Dense)                (None, 2)            10          lwta__dense__activation_46[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "dense_51 (Dense)                (None, 2)            10          lwta__dense__activation_50[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "dense_55 (Dense)                (None, 2)            10          lwta__dense__activation_54[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "dense_59 (Dense)                (None, 2)            10          lwta__dense__activation_58[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "dense_63 (Dense)                (None, 2)            10          lwta__dense__activation_62[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "dense_67 (Dense)                (None, 2)            10          lwta__dense__activation_66[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "dense_71 (Dense)                (None, 2)            10          lwta__dense__activation_70[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "dense_75 (Dense)                (None, 2)            10          lwta__dense__activation_74[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "dense_79 (Dense)                (None, 2)            10          lwta__dense__activation_78[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "lwta__dense__activation_43 (LWT ((None, 2), (None, 1 0           dense_43[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lwta__dense__activation_47 (LWT ((None, 2), (None, 1 0           dense_47[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lwta__dense__activation_51 (LWT ((None, 2), (None, 1 0           dense_51[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lwta__dense__activation_55 (LWT ((None, 2), (None, 1 0           dense_55[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lwta__dense__activation_59 (LWT ((None, 2), (None, 1 0           dense_59[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lwta__dense__activation_63 (LWT ((None, 2), (None, 1 0           dense_63[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lwta__dense__activation_67 (LWT ((None, 2), (None, 1 0           dense_67[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lwta__dense__activation_71 (LWT ((None, 2), (None, 1 0           dense_71[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lwta__dense__activation_75 (LWT ((None, 2), (None, 1 0           dense_75[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lwta__dense__activation_79 (LWT ((None, 2), (None, 1 0           dense_79[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "sb__layer_10 (SB_Layer)         (None, 1)            9           lwta__dense__activation_43[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "sb__layer_11 (SB_Layer)         (None, 1)            9           lwta__dense__activation_47[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "sb__layer_12 (SB_Layer)         (None, 1)            9           lwta__dense__activation_51[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "sb__layer_13 (SB_Layer)         (None, 1)            9           lwta__dense__activation_55[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "sb__layer_14 (SB_Layer)         (None, 1)            9           lwta__dense__activation_59[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "sb__layer_15 (SB_Layer)         (None, 1)            9           lwta__dense__activation_63[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "sb__layer_16 (SB_Layer)         (None, 1)            9           lwta__dense__activation_67[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "sb__layer_17 (SB_Layer)         (None, 1)            9           lwta__dense__activation_71[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "sb__layer_18 (SB_Layer)         (None, 1)            9           lwta__dense__activation_75[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "sb__layer_19 (SB_Layer)         (None, 1)            9           lwta__dense__activation_79[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_20 (Concatenate)    (None, 5)            0           sb__layer_10[0][0]               \n",
      "                                                                 sb__layer_11[0][0]               \n",
      "                                                                 sb__layer_12[0][0]               \n",
      "                                                                 sb__layer_13[0][0]               \n",
      "                                                                 sb__layer_14[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_27 (Concatenate)    (None, 5)            0           sb__layer_15[0][0]               \n",
      "                                                                 sb__layer_16[0][0]               \n",
      "                                                                 sb__layer_17[0][0]               \n",
      "                                                                 sb__layer_18[0][0]               \n",
      "                                                                 sb__layer_19[0][0]               \n",
      "==================================================================================================\n",
      "Total params: 1,198,038\n",
      "Trainable params: 1,195,862\n",
      "Non-trainable params: 2,176\n",
      "__________________________________________________________________________________________________\n",
      "None\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 50000 samples, validate on 10000 samples\n",
      "Epoch 1/300\n",
      "WARNING:tensorflow:From /home/guest/ours/Model_Implementations.py:194: to_float (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.cast` instead.\n",
      "49800/50000 [============================>.] - ETA: 1s - loss: 0.3557 - concatenate_20_loss: 0.1912 - concatenate_27_loss: 0.1756 - concatenate_20_sigmoid_pred: 0.9317 - concatenate_27_sigmoid_pred: 0.9377SAVING WEIGHTS\n",
      "50000/50000 [==============================] - 350s 7ms/sample - loss: 0.3559 - concatenate_20_loss: 0.1912 - concatenate_27_loss: 0.1757 - concatenate_20_sigmoid_pred: 0.9317 - concatenate_27_sigmoid_pred: 0.9378 - val_loss: 0.3808 - val_concatenate_20_loss: 0.1769 - val_concatenate_27_loss: 0.2039 - val_concatenate_20_sigmoid_pred: 0.9431 - val_concatenate_27_sigmoid_pred: 0.9461\n",
      "Epoch 2/300\n",
      "49800/50000 [============================>.] - ETA: 1s - loss: 0.3434 - concatenate_20_loss: 0.1816 - concatenate_27_loss: 0.1729 - concatenate_20_sigmoid_pred: 0.9331 - concatenate_27_sigmoid_pred: 0.9379SAVING WEIGHTS\n",
      "50000/50000 [==============================] - 347s 7ms/sample - loss: 0.3434 - concatenate_20_loss: 0.1816 - concatenate_27_loss: 0.1729 - concatenate_20_sigmoid_pred: 0.9331 - concatenate_27_sigmoid_pred: 0.9380 - val_loss: 0.3330 - val_concatenate_20_loss: 0.1776 - val_concatenate_27_loss: 0.1553 - val_concatenate_20_sigmoid_pred: 0.9455 - val_concatenate_27_sigmoid_pred: 0.9529\n",
      "Epoch 3/300\n",
      "49800/50000 [============================>.] - ETA: 1s - loss: 0.3452 - concatenate_20_loss: 0.1820 - concatenate_27_loss: 0.1744 - concatenate_20_sigmoid_pred: 0.9338 - concatenate_27_sigmoid_pred: 0.9381SAVING WEIGHTS\n",
      "50000/50000 [==============================] - 348s 7ms/sample - loss: 0.3449 - concatenate_20_loss: 0.1820 - concatenate_27_loss: 0.1741 - concatenate_20_sigmoid_pred: 0.9338 - concatenate_27_sigmoid_pred: 0.9382 - val_loss: 0.3408 - val_concatenate_20_loss: 0.1823 - val_concatenate_27_loss: 0.1585 - val_concatenate_20_sigmoid_pred: 0.9437 - val_concatenate_27_sigmoid_pred: 0.9534\n",
      "Epoch 4/300\n",
      "49800/50000 [============================>.] - ETA: 1s - loss: 0.3408 - concatenate_20_loss: 0.1827 - concatenate_27_loss: 0.1693 - concatenate_20_sigmoid_pred: 0.9340 - concatenate_27_sigmoid_pred: 0.9378SAVING WEIGHTS\n",
      "50000/50000 [==============================] - 349s 7ms/sample - loss: 0.3407 - concatenate_20_loss: 0.1826 - concatenate_27_loss: 0.1692 - concatenate_20_sigmoid_pred: 0.9340 - concatenate_27_sigmoid_pred: 0.9378 - val_loss: 0.3171 - val_concatenate_20_loss: 0.1715 - val_concatenate_27_loss: 0.1455 - val_concatenate_20_sigmoid_pred: 0.9468 - val_concatenate_27_sigmoid_pred: 0.9547\n",
      "Epoch 5/300\n",
      "49800/50000 [============================>.] - ETA: 1s - loss: 0.3345 - concatenate_20_loss: 0.1800 - concatenate_27_loss: 0.1658 - concatenate_20_sigmoid_pred: 0.9343 - concatenate_27_sigmoid_pred: 0.9396SAVING WEIGHTS\n",
      "50000/50000 [==============================] - 349s 7ms/sample - loss: 0.3348 - concatenate_20_loss: 0.1801 - concatenate_27_loss: 0.1660 - concatenate_20_sigmoid_pred: 0.9343 - concatenate_27_sigmoid_pred: 0.9396 - val_loss: 0.3261 - val_concatenate_20_loss: 0.1760 - val_concatenate_27_loss: 0.1501 - val_concatenate_20_sigmoid_pred: 0.9456 - val_concatenate_27_sigmoid_pred: 0.9556\n",
      "Epoch 6/300\n",
      "49800/50000 [============================>.] - ETA: 1s - loss: 0.3317 - concatenate_20_loss: 0.1781 - concatenate_27_loss: 0.1650 - concatenate_20_sigmoid_pred: 0.9349 - concatenate_27_sigmoid_pred: 0.9399SAVING WEIGHTS\n",
      "50000/50000 [==============================] - 348s 7ms/sample - loss: 0.3314 - concatenate_20_loss: 0.1779 - concatenate_27_loss: 0.1648 - concatenate_20_sigmoid_pred: 0.9349 - concatenate_27_sigmoid_pred: 0.9399 - val_loss: 0.3130 - val_concatenate_20_loss: 0.1700 - val_concatenate_27_loss: 0.1430 - val_concatenate_20_sigmoid_pred: 0.9428 - val_concatenate_27_sigmoid_pred: 0.9557\n",
      "Epoch 7/300\n",
      "49800/50000 [============================>.] - ETA: 1s - loss: 0.3266 - concatenate_20_loss: 0.1741 - concatenate_27_loss: 0.1640 - concatenate_20_sigmoid_pred: 0.9357 - concatenate_27_sigmoid_pred: 0.9407SAVING WEIGHTS\n",
      "50000/50000 [==============================] - 349s 7ms/sample - loss: 0.3266 - concatenate_20_loss: 0.1740 - concatenate_27_loss: 0.1640 - concatenate_20_sigmoid_pred: 0.9358 - concatenate_27_sigmoid_pred: 0.9406 - val_loss: 0.3168 - val_concatenate_20_loss: 0.1763 - val_concatenate_27_loss: 0.1405 - val_concatenate_20_sigmoid_pred: 0.9449 - val_concatenate_27_sigmoid_pred: 0.9574\n",
      "Epoch 8/300\n",
      "49800/50000 [============================>.] - ETA: 1s - loss: 0.3218 - concatenate_20_loss: 0.1732 - concatenate_27_loss: 0.1601 - concatenate_20_sigmoid_pred: 0.9356 - concatenate_27_sigmoid_pred: 0.9415SAVING WEIGHTS\n",
      "50000/50000 [==============================] - 349s 7ms/sample - loss: 0.3218 - concatenate_20_loss: 0.1732 - concatenate_27_loss: 0.1601 - concatenate_20_sigmoid_pred: 0.9356 - concatenate_27_sigmoid_pred: 0.9415 - val_loss: 0.3151 - val_concatenate_20_loss: 0.1648 - val_concatenate_27_loss: 0.1503 - val_concatenate_20_sigmoid_pred: 0.9466 - val_concatenate_27_sigmoid_pred: 0.9571\n",
      "Epoch 9/300\n",
      "26600/50000 [==============>...............] - ETA: 2:31 - loss: 0.3260 - concatenate_20_loss: 0.1725 - concatenate_27_loss: 0.1651 - concatenate_20_sigmoid_pred: 0.9368 - concatenate_27_sigmoid_pred: 0.9413SAVING WEIGHTS\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-2-2c0855580fae>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     91\u001b[0m \u001b[0mparams_dict\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m'BATCH_NORMALIZATION_FLAG'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mBATCH_NORMALIZATION_FLAG\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'DATA_AUGMENTATION_FLAG'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mDATA_AUGMENTATION_FLAG\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'M'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mM\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'base_model'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mbase_model\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'num_chunks'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mnum_chunks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'model_rep'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mmodel_rep_ens\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'output_activation'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0moutput_activation\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'num_filters_ens'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mnum_filters_ens\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'num_filters_ens_2'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mnum_filters_ens_2\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'batch_size'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'epochs'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mepochs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'dropout_rate'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mdropout_rate_ens\u001b[0m\u001b[0;34m,\u001b[0m  \u001b[0;34m'lr'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mlr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'blend_factor'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mblend_factor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'inp_shape'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0minp_shape\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'noise_stddev'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mnoise_stddev\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'weight_save_freq'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mweight_save_freq\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'name'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'model_path'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mmodel_path\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     92\u001b[0m \u001b[0mm3\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mModel_Logistic_Ensemble\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparams_dict\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 93\u001b[0;31m \u001b[0mm3\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdefineModel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m;\u001b[0m \u001b[0mm3\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrainModel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     94\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     95\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/ours/Model.py\u001b[0m in \u001b[0;36mtrainModel\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    240\u001b[0m                         \u001b[0mshuffle\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    241\u001b[0m                         \u001b[0mvalidation_data\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata_dict\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'X_test'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mY_test_list\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 242\u001b[0;31m                         callbacks=[WS])\n\u001b[0m\u001b[1;32m    243\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    244\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_freq, max_queue_size, workers, use_multiprocessing, **kwargs)\u001b[0m\n\u001b[1;32m    725\u001b[0m         \u001b[0mmax_queue_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmax_queue_size\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    726\u001b[0m         \u001b[0mworkers\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mworkers\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 727\u001b[0;31m         use_multiprocessing=use_multiprocessing)\n\u001b[0m\u001b[1;32m    728\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    729\u001b[0m   def evaluate(self,\n",
      "\u001b[0;32m~/.local/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/training_v2.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, model, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_freq, max_queue_size, workers, use_multiprocessing, **kwargs)\u001b[0m\n\u001b[1;32m    329\u001b[0m                 \u001b[0mmode\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mModeKeys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTRAIN\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    330\u001b[0m                 \u001b[0mtraining_context\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtraining_context\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 331\u001b[0;31m                 total_epochs=epochs)\n\u001b[0m\u001b[1;32m    332\u001b[0m             \u001b[0mcbks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmake_logs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepoch_logs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtraining_result\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mModeKeys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTRAIN\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    333\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/training_v2.py\u001b[0m in \u001b[0;36mrun_one_epoch\u001b[0;34m(model, iterator, execution_function, dataset_size, batch_size, strategy, steps_per_epoch, num_samples, mode, training_context, total_epochs)\u001b[0m\n\u001b[1;32m    121\u001b[0m         step=step, mode=mode, size=current_batch_size) as batch_logs:\n\u001b[1;32m    122\u001b[0m       \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 123\u001b[0;31m         \u001b[0mbatch_outs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mexecution_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miterator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    124\u001b[0m       \u001b[0;32mexcept\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mStopIteration\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOutOfRangeError\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    125\u001b[0m         \u001b[0;31m# TODO(kaftan): File bug about tf function and errors.OutOfRangeError?\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/training_v2_utils.py\u001b[0m in \u001b[0;36mexecution_function\u001b[0;34m(input_fn)\u001b[0m\n\u001b[1;32m     84\u001b[0m     \u001b[0;31m# `numpy` translates Tensors to values in Eager mode.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     85\u001b[0m     return nest.map_structure(_non_none_constant_value,\n\u001b[0;32m---> 86\u001b[0;31m                               distributed_function(input_fn))\n\u001b[0m\u001b[1;32m     87\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     88\u001b[0m   \u001b[0;32mreturn\u001b[0m \u001b[0mexecution_function\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/training_v2_utils.py\u001b[0m in \u001b[0;36mdistributed_function\u001b[0;34m(input_iterator)\u001b[0m\n\u001b[1;32m     71\u001b[0m     \u001b[0mstrategy\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdistribution_strategy_context\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_strategy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     72\u001b[0m     outputs = strategy.experimental_run_v2(\n\u001b[0;32m---> 73\u001b[0;31m         per_replica_function, args=(x, y, sample_weights))\n\u001b[0m\u001b[1;32m     74\u001b[0m     \u001b[0;31m# Out of PerReplica outputs reduce or pick values to return.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     75\u001b[0m     all_outputs = dist_utils.unwrap_output_dict(\n",
      "\u001b[0;32m~/.local/lib/python3.6/site-packages/tensorflow_core/python/distribute/distribute_lib.py\u001b[0m in \u001b[0;36mexperimental_run_v2\u001b[0;34m(self, fn, args, kwargs)\u001b[0m\n\u001b[1;32m    762\u001b[0m       fn = autograph.tf_convert(fn, ag_ctx.control_status_ctx(),\n\u001b[1;32m    763\u001b[0m                                 convert_by_default=False)\n\u001b[0;32m--> 764\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_extended\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcall_for_each_replica\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    765\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    766\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0mreduce\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreduce_op\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.6/site-packages/tensorflow_core/python/distribute/distribute_lib.py\u001b[0m in \u001b[0;36mcall_for_each_replica\u001b[0;34m(self, fn, args, kwargs)\u001b[0m\n\u001b[1;32m   1808\u001b[0m       \u001b[0mkwargs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1809\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_container_strategy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mscope\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1810\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call_for_each_replica\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1811\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1812\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_call_for_each_replica\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.6/site-packages/tensorflow_core/python/distribute/distribute_lib.py\u001b[0m in \u001b[0;36m_call_for_each_replica\u001b[0;34m(self, fn, args, kwargs)\u001b[0m\n\u001b[1;32m   2153\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_container_strategy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2154\u001b[0m         replica_id_in_sync_group=constant_op.constant(0, dtypes.int32)):\n\u001b[0;32m-> 2155\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2156\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2157\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_reduce_to\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreduce_op\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdestinations\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.6/site-packages/tensorflow_core/python/autograph/impl/api.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    256\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    257\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0mag_ctx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mControlStatusCtx\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstatus\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mag_ctx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mStatus\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mUNSPECIFIED\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 258\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    259\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    260\u001b[0m   \u001b[0;32mif\u001b[0m \u001b[0minspect\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0misfunction\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0minspect\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mismethod\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/training_v2_utils.py\u001b[0m in \u001b[0;36mtrain_on_batch\u001b[0;34m(model, x, y, sample_weight, class_weight, reset_metrics)\u001b[0m\n\u001b[1;32m    262\u001b[0m       \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    263\u001b[0m       \u001b[0msample_weights\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msample_weights\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 264\u001b[0;31m       output_loss_metrics=model._output_loss_metrics)\n\u001b[0m\u001b[1;32m    265\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    266\u001b[0m   \u001b[0;32mif\u001b[0m \u001b[0mreset_metrics\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/training_eager.py\u001b[0m in \u001b[0;36mtrain_on_batch\u001b[0;34m(model, inputs, targets, sample_weights, output_loss_metrics)\u001b[0m\n\u001b[1;32m    309\u001b[0m           \u001b[0msample_weights\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msample_weights\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    310\u001b[0m           \u001b[0mtraining\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 311\u001b[0;31m           output_loss_metrics=output_loss_metrics))\n\u001b[0m\u001b[1;32m    312\u001b[0m   \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    313\u001b[0m     \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/training_eager.py\u001b[0m in \u001b[0;36m_process_single_batch\u001b[0;34m(model, inputs, targets, output_loss_metrics, sample_weights, training)\u001b[0m\n\u001b[1;32m    266\u001b[0m           \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_backwards\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtape\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mscaled_total_loss\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    267\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 268\u001b[0;31m           \u001b[0mgrads\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtape\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgradient\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mscaled_total_loss\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrainable_weights\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    269\u001b[0m           if isinstance(model.optimizer,\n\u001b[1;32m    270\u001b[0m                         loss_scale_optimizer.LossScaleOptimizer):\n",
      "\u001b[0;32m~/.local/lib/python3.6/site-packages/tensorflow_core/python/eager/backprop.py\u001b[0m in \u001b[0;36mgradient\u001b[0;34m(self, target, sources, output_gradients, unconnected_gradients)\u001b[0m\n\u001b[1;32m   1013\u001b[0m         \u001b[0moutput_gradients\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0moutput_gradients\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1014\u001b[0m         \u001b[0msources_raw\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mflat_sources_raw\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1015\u001b[0;31m         unconnected_gradients=unconnected_gradients)\n\u001b[0m\u001b[1;32m   1016\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1017\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_persistent\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.6/site-packages/tensorflow_core/python/eager/imperative_grad.py\u001b[0m in \u001b[0;36mimperative_grad\u001b[0;34m(tape, target, sources, output_gradients, sources_raw, unconnected_gradients)\u001b[0m\n\u001b[1;32m     74\u001b[0m       \u001b[0moutput_gradients\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     75\u001b[0m       \u001b[0msources_raw\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 76\u001b[0;31m       compat.as_str(unconnected_gradients.value))\n\u001b[0m",
      "\u001b[0;32m~/.local/lib/python3.6/site-packages/tensorflow_core/python/eager/backprop.py\u001b[0m in \u001b[0;36m_gradient_function\u001b[0;34m(op_name, attr_tuple, num_inputs, inputs, outputs, out_grads, skip_input_indices)\u001b[0m\n\u001b[1;32m    136\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mnum_inputs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    137\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 138\u001b[0;31m   \u001b[0;32mreturn\u001b[0m \u001b[0mgrad_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmock_op\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0mout_grads\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    139\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    140\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.6/site-packages/tensorflow_core/python/ops/math_grad.py\u001b[0m in \u001b[0;36m_RealDivGrad\u001b[0;34m(op, grad)\u001b[0m\n\u001b[1;32m   1276\u001b[0m   \u001b[0my\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mop\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1277\u001b[0m   \u001b[0msx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0marray_ops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1278\u001b[0;31m   \u001b[0msy\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0marray_ops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1279\u001b[0m   \u001b[0mrx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mry\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgen_array_ops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbroadcast_gradient_args\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msy\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1280\u001b[0m   \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmath_ops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconj\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.6/site-packages/tensorflow_core/python/ops/array_ops.py\u001b[0m in \u001b[0;36mshape\u001b[0;34m(input, name, out_type)\u001b[0m\n\u001b[1;32m    445\u001b[0m     \u001b[0mA\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m`\u001b[0m\u001b[0mTensor\u001b[0m\u001b[0;31m`\u001b[0m \u001b[0mof\u001b[0m \u001b[0mtype\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m`\u001b[0m\u001b[0mout_type\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    446\u001b[0m   \"\"\"\n\u001b[0;32m--> 447\u001b[0;31m   \u001b[0;32mreturn\u001b[0m \u001b[0mshape_internal\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptimize\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mout_type\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mout_type\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    448\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    449\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.6/site-packages/tensorflow_core/python/ops/array_ops.py\u001b[0m in \u001b[0;36mshape_internal\u001b[0;34m(input, name, optimize, out_type)\u001b[0m\n\u001b[1;32m    473\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0moptimize\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0minput_shape\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_fully_defined\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    474\u001b[0m           \u001b[0;32mreturn\u001b[0m \u001b[0mconstant\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput_shape\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_list\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mout_type\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 475\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mgen_array_ops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mout_type\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mout_type\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    476\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    477\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.6/site-packages/tensorflow_core/python/ops/gen_array_ops.py\u001b[0m in \u001b[0;36mshape\u001b[0;34m(input, out_type, name)\u001b[0m\n\u001b[1;32m   8957\u001b[0m       _result = _pywrap_tensorflow.TFE_Py_FastPathExecute(\n\u001b[1;32m   8958\u001b[0m         \u001b[0m_ctx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_context_handle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_ctx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_thread_local_data\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdevice_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"Shape\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 8959\u001b[0;31m         name, _ctx.post_execution_callbacks, input, \"out_type\", out_type)\n\u001b[0m\u001b[1;32m   8960\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0m_result\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   8961\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0m_core\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_FallbackException\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "#GENERAL PARAMETERS - SET THESE APPROPRIATELY\n",
    "model_path = 'models/'  #path to save model weights to\n",
    "weight_save_freq = 1  #how frequently (in epochs, e.g. every 10 epochs) to save weights to disk\n",
    "tf.set_random_seed(1) \n",
    "\n",
    "\n",
    "# #######DATASET-SPECIFIC PARAMETERS: CHOOSE THIS BLOCK FOR MNIST\n",
    "# DATA_DESC = 'MNIST'; (X_train, Y_train), (X_test, Y_test) = mnist.load_data()\n",
    "# Y_train = np.squeeze(Y_train); Y_test = np.squeeze(Y_test)\n",
    "# num_channels = 1; inp_shape = (28,28,1); num_classes=10\n",
    "# #MODEL-SPECIFIC PARAMETERS: MNIST\n",
    "# #PARAMETERS RELATED TO SGD OPTIMIZATION\n",
    "# epochs=150; batch_size=200; lr=3e-4; \n",
    "# #MODEL DEFINTION PARAMETERS\n",
    "# num_filters_std = [64, 64, 64]; num_filters_ens=[32, 32, 32]; num_filters_ens_2=4; \n",
    "# dropout_rate_std=0.0; dropout_rate_ens=0.0; weight_decay = 0 \n",
    "# noise_stddev = 0.3; blend_factor=0.3; \n",
    "# model_rep_baseline=1; model_rep_ens=2; \n",
    "# DATA_AUGMENTATION_FLAG=0; BATCH_NORMALIZATION_FLAG=0\n",
    "# #######END: DATASET-SPECIFIC PARAMETERS: MNIST\n",
    "\n",
    "\n",
    "#########DATASET-SPECIFIC PARAMETERS: CHOOSE THIS BLOCK FOR CIFAR10\n",
    "DATA_DESC = 'CIFAR10'; (X_train, Y_train), (X_test, Y_test) = cifar10.load_data()\n",
    "Y_train = np.squeeze(Y_train); Y_test = np.squeeze(Y_test)\n",
    "num_channels = 3; inp_shape = (32,32,3); num_classes=10\n",
    "#MODEL-SPECIFIC PARAMETERS: CIFAR10\n",
    "#PARAMETERS RELATED TO SGD OPTIMIZATION\n",
    "epochs=300; batch_size=200; lr=2e-4; \n",
    "#MODEL DEFINTION PARAMETERS\n",
    "num_filters_std = [32, 64, 128]; num_filters_ens=[32, 64, 128]; num_filters_ens_2=16; \n",
    "dropout_rate_std=0.0; dropout_rate_ens=0.0; weight_decay = 0 \n",
    "noise_stddev = 0.032; blend_factor=0.032; \n",
    "model_rep_baseline=2; model_rep_ens=2; \n",
    "DATA_AUGMENTATION_FLAG=1; BATCH_NORMALIZATION_FLAG=1\n",
    "##########END: DATASET-SPECIFIC PARAMETERS: CIFAR10\n",
    "\n",
    "\n",
    "\n",
    "# DATA PRE-PROCESSING\n",
    "X_train = (X_train/255).astype(np.float32);  X_test = (X_test/255).astype(np.float32); #scale data to (0,1)\n",
    "X_train = X_train.reshape(X_train.shape[0], X_train.shape[1], X_train.shape[2],num_channels); X_test = X_test.reshape(X_test.shape[0], X_test.shape[1], X_test.shape[2],num_channels)\n",
    "X_valid = X_test[0:1000]; Y_valid = Y_test[0:1000]; #validation data (to monitor accuracy during training)\n",
    "X_train = X_train-0.5; X_test = X_test-0.5; X_valid = X_valid-0.5; #map to range (-0.5,0.5)\n",
    "data_dict = {'X_train':X_train, 'Y_train_cat':Y_train, 'X_test':X_test, 'Y_test_cat':Y_test}\n",
    "\n",
    "\n",
    "\n",
    "### TRAIN MODEL. each block below corresponds to one of the models in Table 1 of the paper. In order to train, \n",
    "#   uncomment the final two lines of the block of interest and then run this script\n",
    "\n",
    "\n",
    "### BASELINE SOFTMAX MODEL DEFINITION\n",
    "name = 'softmax_baseline'+'_'+DATA_DESC; num_chunks=1\n",
    "M = np.eye(num_classes).astype(np.float32)\n",
    "output_activation = 'softmax'; base_model=None\n",
    "params_dict = {'weight_decay':weight_decay, 'num_filters_std':num_filters_std, 'BATCH_NORMALIZATION_FLAG':BATCH_NORMALIZATION_FLAG, 'DATA_AUGMENTATION_FLAG':DATA_AUGMENTATION_FLAG, 'M':M, 'model_rep':model_rep_baseline, 'base_model':base_model, 'num_chunks':num_chunks, 'output_activation':output_activation,  'batch_size':batch_size, 'epochs':epochs, 'lr':lr, 'dropout_rate':dropout_rate_std,  'blend_factor':blend_factor, 'inp_shape':inp_shape, 'noise_stddev':noise_stddev, 'weight_save_freq':weight_save_freq, 'name':name, 'model_path':model_path}\n",
    "# m0 = Model_Softmax_Baseline(data_dict, params_dict)\n",
    "# m0.defineModel(); m0.trainModel()\n",
    "\n",
    "\n",
    "## BASELINE LOGISTIC MODEL DEFINITION\n",
    "name = 'logistic_baseline'+'_'+DATA_DESC; num_chunks=1\n",
    "M = np.eye(num_classes).astype(np.float32)\n",
    "output_activation = 'sigmoid'; base_model=None\n",
    "params_dict = {'weight_decay':weight_decay, 'num_filters_std':num_filters_std, 'BATCH_NORMALIZATION_FLAG':BATCH_NORMALIZATION_FLAG, 'DATA_AUGMENTATION_FLAG':DATA_AUGMENTATION_FLAG, 'M':M, 'model_rep':model_rep_baseline, 'base_model':base_model, 'num_chunks':num_chunks, 'output_activation':output_activation,  'batch_size':batch_size, 'epochs':epochs, 'lr':lr, 'dropout_rate':dropout_rate_std,  'blend_factor':blend_factor, 'inp_shape':inp_shape, 'noise_stddev':noise_stddev, 'weight_save_freq':weight_save_freq, 'name':name, 'model_path':model_path}\n",
    "# m1 = Model_Logistic_Baseline(data_dict, params_dict)\n",
    "# m1.defineModel(); m1.trainModel()\n",
    "\n",
    "\n",
    "## BASELINE TANH MODEL DEFINITION\n",
    "name = 'Tanh_baseline_16'+'_'+DATA_DESC; seed = 59; num_chunks=1; code_length=16; num_codes=num_classes; code_length_true=code_length\n",
    "M = scipy.linalg.hadamard(code_length).astype(np.float32)\n",
    "M[np.arange(0, num_codes,2), 0]= -1#replace first col, which for this Hadamard construction is always 1, hence not a useful bit\n",
    "np.random.seed(seed); np.random.shuffle(M)\n",
    "idx=np.random.permutation(code_length)\n",
    "M = M[0:num_codes, idx[0:code_length_true]]\n",
    "base_model=None\n",
    "def output_activation(x):\n",
    "    return tf.nn.tanh(x)\n",
    "params_dict = {'weight_decay':weight_decay, 'num_filters_std':num_filters_std, 'BATCH_NORMALIZATION_FLAG':BATCH_NORMALIZATION_FLAG, 'DATA_AUGMENTATION_FLAG':DATA_AUGMENTATION_FLAG, 'M':M, 'model_rep':model_rep_baseline, 'base_model':base_model, 'num_chunks':num_chunks, 'output_activation':output_activation,  'batch_size':batch_size, 'epochs':epochs, 'dropout_rate':dropout_rate_std,  'lr':lr, 'blend_factor':blend_factor, 'inp_shape':inp_shape, 'noise_stddev':noise_stddev, 'weight_save_freq':weight_save_freq, 'name':name, 'model_path':model_path}\n",
    "# m2 = Model_Tanh_Baseline(data_dict, params_dict)\n",
    "# m2.defineModel(); m2.trainModel()\n",
    "\n",
    "## ENSEMBLE LOGISTIC MODEL DEFINITION\n",
    "name = 'logistic_diverse'+'_'+DATA_DESC; num_chunks=2\n",
    "M = np.eye(num_classes).astype(np.float32)\n",
    "base_model=None\n",
    "def output_activation(x):\n",
    "    return tf.nn.sigmoid(x)\n",
    "params_dict = {'BATCH_NORMALIZATION_FLAG':BATCH_NORMALIZATION_FLAG, 'DATA_AUGMENTATION_FLAG':DATA_AUGMENTATION_FLAG, 'M':M, 'base_model':base_model, 'num_chunks':num_chunks, 'model_rep': model_rep_ens, 'output_activation':output_activation, 'num_filters_ens':num_filters_ens, 'num_filters_ens_2':num_filters_ens_2,'batch_size':batch_size, 'epochs':epochs, 'dropout_rate':dropout_rate_ens,  'lr':lr, 'blend_factor':blend_factor, 'inp_shape':inp_shape, 'noise_stddev':noise_stddev, 'weight_save_freq':weight_save_freq, 'name':name, 'model_path':model_path}\n",
    "m3 = Model_Logistic_Ensemble(data_dict, params_dict)\n",
    "m3.defineModel(); m3.trainModel()\n",
    "\n",
    "\n",
    "\n",
    "#COMMENTS FOR ALL TANH ENSEMBLE MODELS: \n",
    "#1. num_chunks refers to how many models comprise the ensemble (4 is used in the paper); code_length/num_chunks shoould be an integer\n",
    "#2. output_activation is the function to apply to the logits\n",
    "#   a. one can use anything which gives support to positive and negative values (since output code has +1/-1 elements); tanh or identity maps both work\n",
    "#   b. in order to alleviate potential concerns of gradient masking with tanh, one can use identity as well\n",
    "#3. M is the actual coding matrix (referred to in the paper as H).  Each row is a codeword\n",
    "#   note that any random shuffle of a Hadmard matrix's rows or columns is still orthogonal\n",
    "#4. There is nothing particularly special about the seed (which effectively determines the coding matrix). \n",
    "#   We tried several seeds from 0-60 and found that all give comparable model performance (e.g. benign and adversarial accuracy). \n",
    "\n",
    "## ENSEMBLE TANH 16 MODEL DEFINITION\n",
    "name = 'tanh_16_diverse'+'_'+DATA_DESC; seed = 59; code_length=16; num_codes=code_length; num_chunks=4; base_model=None; \n",
    "def output_activation(x):\n",
    "    return tf.nn.tanh(x)\n",
    "M = scipy.linalg.hadamard(code_length).astype(np.float32)\n",
    "M[np.arange(0, num_codes,2), 0]= -1#replace first col, which for scipy's Hadamard construction is always 1, hence not a useful classifier; this change still ensures all codewords have dot product <=0; since our decoder ignores negative correlations anyway, this has no net effect on probability estimation\n",
    "np.random.seed(seed)\n",
    "np.random.shuffle(M)\n",
    "idx=np.random.permutation(code_length)\n",
    "M = M[0:num_codes, idx[0:code_length]]\n",
    "params_dict = {'BATCH_NORMALIZATION_FLAG':BATCH_NORMALIZATION_FLAG, 'DATA_AUGMENTATION_FLAG':DATA_AUGMENTATION_FLAG, 'M':M, 'base_model':base_model, 'num_chunks':num_chunks, 'model_rep': model_rep_ens, 'output_activation':output_activation, 'num_filters_ens':num_filters_ens, 'num_filters_ens_2':num_filters_ens_2,'batch_size':batch_size, 'epochs':epochs, 'dropout_rate':dropout_rate_ens,  'lr':lr, 'blend_factor':blend_factor, 'inp_shape':inp_shape, 'noise_stddev':noise_stddev, 'weight_save_freq':weight_save_freq, 'name':name, 'model_path':model_path}\n",
    "# m4 = Model_Tanh_Ensemble(data_dict, params_dict)\n",
    "# m4.defineModel(); m4.trainModel()\n",
    "\n",
    "\n",
    "\n",
    "## ENSEMBLE TANH 32 MODEL DEFINITION\n",
    "name = 'tanh_32_diverse'+'_'+DATA_DESC; seed = 59; code_length=32; num_codes=code_length; num_chunks=4; base_model=None;\n",
    "def output_activation(x):\n",
    "    return tf.nn.tanh(x)\n",
    "M = scipy.linalg.hadamard(code_length).astype(np.float32)\n",
    "M[np.arange(0, num_codes,2), 0]= -1#replace first col, which for scipy's Hadamard construction is always 1, hence not a useful classifier; this change still ensures all codewords have dot product <=0; since our decoder ignores negative correlations anyway, this has no net effect on probability estimation\n",
    "np.random.seed(seed)\n",
    "np.random.shuffle(M)\n",
    "idx=np.random.permutation(code_length)\n",
    "M = M[0:num_codes, idx[0:code_length]]\n",
    "params_dict = {'BATCH_NORMALIZATION_FLAG':BATCH_NORMALIZATION_FLAG, 'DATA_AUGMENTATION_FLAG':DATA_AUGMENTATION_FLAG, 'M':M, 'base_model':base_model, 'num_chunks':num_chunks, 'model_rep': model_rep_ens, 'output_activation':output_activation, 'num_filters_ens':num_filters_ens, 'num_filters_ens_2':num_filters_ens_2,'batch_size':batch_size, 'epochs':epochs, 'dropout_rate':dropout_rate_ens,  'lr':lr, 'blend_factor':blend_factor, 'inp_shape':inp_shape, 'noise_stddev':noise_stddev, 'weight_save_freq':weight_save_freq, 'name':name, 'model_path':model_path}\n",
    "# m5 = Model_Tanh_Ensemble(data_dict, params_dict)\n",
    "# m5.defineModel();   m5.trainModel()\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "adver",
   "language": "python",
   "name": "adver"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}