3.1 Deconvolution.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3.1 Deconvolution"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tensorflow Walkthrough"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. Import Dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting MNIST_data/train-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data/train-labels-idx1-ubyte.gz\n",
      "Extracting MNIST_data/t10k-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "from tensorflow.python.ops import nn_ops, gen_nn_ops\n",
    "from tensorflow.python.framework import ops\n",
    "import matplotlib.pyplot as plt\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "\n",
    "from models.models_3_1 import MNIST_CNN\n",
    "from utils import pixel_range\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "mnist = input_data.read_data_sets(\"MNIST_data/\", one_hot=True)\n",
    "\n",
    "images = mnist.train.images\n",
    "labels = mnist.train.labels\n",
    "\n",
    "logdir = './tf_logs/3_1_DC/'\n",
    "ckptdir = logdir + 'model'\n",
    "\n",
    "if not os.path.exists(logdir):\n",
    "    os.mkdir(logdir)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. Building Graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with tf.name_scope('Classifier'):\n",
    "\n",
    "    # Initialize neural network\n",
    "    DNN = MNIST_CNN('CNN')\n",
    "\n",
    "    # Setup training process\n",
    "    X = tf.placeholder(tf.float32, [None, 784], name='X')\n",
    "    Y = tf.placeholder(tf.float32, [None, 10], name='Y')\n",
    "\n",
    "    activations, logits = DNN(X)\n",
    "    \n",
    "    tf.add_to_collection('DC', X)\n",
    "    \n",
    "    for activation in activations:\n",
    "        tf.add_to_collection('DC', activation)\n",
    "\n",
    "    cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=Y))\n",
    "\n",
    "    optimizer = tf.train.AdamOptimizer().minimize(cost, var_list=DNN.vars)\n",
    "\n",
    "    correct_prediction = tf.equal(tf.argmax(logits, 1), tf.argmax(Y, 1))\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "\n",
    "cost_summary = tf.summary.scalar('Cost', cost)\n",
    "accuray_summary = tf.summary.scalar('Accuracy', accuracy)\n",
    "summary = tf.summary.merge_all()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. Training Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0001 cost = 0.156623523 accuracy = 0.950163645\n",
      "Epoch: 0002 cost = 0.042625537 accuracy = 0.986927283\n",
      "Epoch: 0003 cost = 0.029875092 accuracy = 0.990618190\n",
      "Epoch: 0004 cost = 0.021823779 accuracy = 0.993036370\n",
      "Epoch: 0005 cost = 0.018047505 accuracy = 0.993927278\n",
      "Epoch: 0006 cost = 0.013701927 accuracy = 0.995400004\n",
      "Epoch: 0007 cost = 0.012501947 accuracy = 0.996018185\n",
      "Epoch: 0008 cost = 0.010178409 accuracy = 0.996618185\n",
      "Epoch: 0009 cost = 0.008392057 accuracy = 0.997272730\n",
      "Epoch: 0010 cost = 0.008140020 accuracy = 0.997181821\n",
      "Epoch: 0011 cost = 0.007686417 accuracy = 0.997581820\n",
      "Epoch: 0012 cost = 0.006270998 accuracy = 0.998145456\n",
      "Epoch: 0013 cost = 0.004811549 accuracy = 0.998509092\n",
      "Epoch: 0014 cost = 0.007982755 accuracy = 0.997454548\n",
      "Epoch: 0015 cost = 0.004931021 accuracy = 0.998309093\n",
      "Accuracy: 0.9935\n"
     ]
    }
   ],
   "source": [
    "sess = tf.InteractiveSession()\n",
    "sess.run(tf.global_variables_initializer())\n",
    "\n",
    "saver = tf.train.Saver()\n",
    "file_writer = tf.summary.FileWriter(logdir, tf.get_default_graph())\n",
    "\n",
    "# Hyper parameters\n",
    "training_epochs = 15\n",
    "batch_size = 100\n",
    "\n",
    "for epoch in range(training_epochs):\n",
    "    total_batch = int(mnist.train.num_examples / batch_size)\n",
    "    avg_cost = 0\n",
    "    avg_acc = 0\n",
    "    \n",
    "    for i in range(total_batch):\n",
    "        batch_xs, batch_ys = mnist.train.next_batch(batch_size)\n",
    "        _, c, a, summary_str = sess.run([optimizer, cost, accuracy, summary], feed_dict={X: batch_xs, Y: batch_ys})\n",
    "        avg_cost += c / total_batch\n",
    "        avg_acc += a / total_batch\n",
    "        \n",
    "        file_writer.add_summary(summary_str, epoch * total_batch + i)\n",
    "\n",
    "    print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.9f}'.format(avg_cost), 'accuracy =', '{:.9f}'.format(avg_acc))\n",
    "    \n",
    "    saver.save(sess, ckptdir)\n",
    "\n",
    "print('Accuracy:', sess.run(accuracy, feed_dict={X: mnist.test.images, Y: mnist.test.labels}))\n",
    "\n",
    "sess.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4. Restoring Subgraph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./tf_logs/2_3_DC/model\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "@ops.RegisterGradient(\"DeconvRelu\")\n",
    "def _DeconvReluGrad(op, grad):\n",
    "    return tf.where(0. < grad, grad, tf.zeros(tf.shape(grad)))\n",
    "\n",
    "sess = tf.InteractiveSession()\n",
    "\n",
    "g = tf.get_default_graph()\n",
    "with g.gradient_override_map({'Relu': 'DeconvRelu'}):\n",
    "    new_saver = tf.train.import_meta_graph(ckptdir + '.meta')\n",
    "\n",
    "new_saver.restore(sess, tf.train.latest_checkpoint(logdir))\n",
    "\n",
    "activations = tf.get_collection('DC')\n",
    "weights = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope='CNN')\n",
    "\n",
    "X = activations[0]\n",
    "activations = activations[1:]\n",
    "\n",
    "sample_imgs = [images[np.argmax(labels, axis=1) == i][5] for i in range(10)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 5. Displaying Images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABDAAAADCCAYAAAC/gTIXAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xm0XWWd5vHfe6+EkBBIcjNPN3NCmEKYFIyKICiTlF2U\nigPR0qpqa7B0aZXlsFqrpMt2tdVWdS27y6FbWi2HFoqhRROhQAxIoiSECyFkTm7meSBJAcl9+49z\nqbqV/TzJ2ckJ2ffm+1mLtfSXzT57ePe7dzbnPL+Ucw4AAAAAAIAqazrZGwAAAAAAAHA0vMAAAAAA\nAACVxwsMAAAAAABQebzAAAAAAAAAlccLDAAAAAAAUHm8wAAAAAAAAJXHC4yKSyn9z5TS5xq9LAAA\nAAAA3ckp9wIjpbQ6pXTNq/A5n08pfbeObTmQUtqbUtqVUno8pfQHKaV/PS855z/IOf9VPZ/ZddmU\n0ptSSuuOby/QHXQZRy90+WfEca7zVR8/KaWrUkoPp5R2p5RWv5qfjZOjp4zdLp/dK6X03OGfn1LK\nKaV9Xfbxmydj+3Bi9JRxnFL66WH78FJKqa3Lnx++n3Neze1DY/WgcXvEZ4eU0l+llNpSSgdTSp8X\n/25b5zP49pTSP6WURr5a247jdyqM45TSkJTS91NKGzr//LGU0uVd/nx4Sum+zj/PKaWxr+a2nwyn\n3AuMCrop59wvIloj4ksR8ecR8a2Tu0nohm7KOZ/Z5Z8NJ3NjUkqvOYZ/bV9E/K+I+GSDNwfV1hPG\n7is+GRFbzZ9d2GUfP3Qcn4Fq6vbjOOf8tq77EBGPR8T/PWyxrvt5bUM2FidTtx+3cfRnh+UR8WcR\n8RPxZ4sj4rqcc/+IGBERyyLifxzDNuDk6unj+MyI+HVEXBwRAyPizoj4SUrpzM4/74iIn0XEfziG\nz+2WTukXGCmlWSmluSml/5pS2plSWpVSeluXP38kpfTXKaX5KaU9KaV7U0oDO/+s8Hau8y3gNSml\nt0bEpyPinZ1vAhcdbVtyzrtzzvdFxDsj4vaU0nmd6/x2SumLXT7jz1JKGzvfsn2o803bxK7LppT6\nRsRPI2JEo95GontKKb021b7ZsyultCil9KYuf/aBzv9avDeltDKl9PuddTl+xFj8d9dA5/j/85TS\n0xGxL6X0ms5/766U0tbO6+tP3LbmnOfnnL8TEStPwKFAN9Odxm7nOsZFxHsj4q8beyTQnXW3cdxl\nXWMjYmZE/J9GHAd0L91p3B7t2SHnfGfO+acRsVf82ebD/rJ7KCIm1nucUG09ZRznnFfmnP8m57wx\n53wo5/z1iOgVEVM6/3xzzvlrUXvJcUo4pV9gdLo8Ip6PiEER8eWI+FZKKXX58/dHxAcjYnhEHIyI\nvzvaCnPOP4uI/xwRP+x8E3hhvRuTc54fEeui9uDw76Tai5GPR8Q1UZtg32TWsS8i3hYRG6ryNhKv\nvlT7GuRPIuKLUXtj+4mIuCulNLhzkS0RcWNEnBURH4iI/5ZSmnGc4+fdEXFDRPSP2hvh+yNiUUSM\njIirI+JPU0rXNWQH0WN107H736P24vqA+fNHU0qbUkp3p1Pg653otuP4Fe+PiF/mnFcfVv9e58P4\nnJRS3c826D66+bgtLaU0JqW0K2pz9yei9ncBdHM9eRynlKZH7QXG8uNdV3fFC4yINTnnb+ScD0Xt\nKznDI2Jolz//Ts75mc4B/bmI+J2UUvMJ3qYNUbvYDvc7EfG/c87P5pz3R8TnT/B2oPu4p/MN866U\n0j2dtfdGxAM55wdyzh05559HxG8i4vqIiJzzT3LOK3LNLyJiTogXZyX9Xc65Ped8ICIujYjBOee/\nzDm/lHNeGRHfiIh3HednoGfp9mM3pfRbEdGcc/4ns+43RsTYiJgatfn9/6Xj+6kKqqfbj+PDvD8i\nvn1Y7T1RG8etEfFwRMxOKfU/zu3FydXTxm1pOee1nT8hGRQRn42IJSfic3BCnTLjOKV0VkR8JyK+\nkHPefZzb223xABWx6ZX/kXPe3/nlizO7/Hl7l/+9JiJOi9okdyKNjIgdoj4iahffK9rFMjg13ZJz\nfvCwWmtE3JpSuqlL7bSoPXhGqv1c6j9FxOSovczsExFtcXy6jsnWqH0Fb1eXWnNE/PI4PwM9S7ce\nu51fN/1ydD4UKTnnRzv/50sppY9GxJ6IOKcB24zq6NbjuKuU0usjYlhE/LhrPef8WJf/+9cppduj\n9sB//3FtMU6mHjNuj1fOeUdK6c6IWJRSGplzPngiPw8NdUqM45TSGVGbb5/IOZ/SP1flBcbRje7y\nv8dExMsRsS1qYSt9XvmDzm9lDO6ybD6WD0spXRq1FxhzxR9vjIhRZtsOd0yfjx6lPWrfIPrw4X+Q\nUjo9Iu6K2n9luzfn/HLnW+tXfj6lxs+/G/NRe8A9XNd/rz0iVuWcJx3LxuOU1p3G7qSo/VfpX3a+\nAO8VEWenlDZFxGvFV/Bf+awk6uhZutM47ur2iLg75/zCUZZjHPdM3XXcNsJrImJI1H5WoP5DIrqP\nHjWOO7f5nqjFDPx+I9bZnfETkqN7b0ppWkqpT0T8ZUT8uPPnJksjondK6YaU0mlR+9rZ6V3+vc0R\nMTZ1aYl6JCmls1JKN0bEDyLiuzln9RbwRxHxgZTSOZ3b87kjrHJzRLSklM6u5/PRI303Im5KKV2X\nUmpOKfXuDB0aFbW/ZJ0etY4JBzvfRHdNlFfj56mIuD6lNDClNCwi/vQonz8/IvZ2hhqd0bkN53W+\npCtIKTWllHpH7Q156tzeXse05+juutPYfSZqL5Ond/7zoc5tmB4R7Smlc1NK0zvXcWZEfCUi1kfE\nc+UOCbqh7jSOI+Jf/wvf78RhPx/pzAm4MtVaBfdOKX0yat9GfUysBt1btxq3R3t2SCmd1vnnTRHx\nms4/b+78s3eklKZ0rmNwRPxNRCzMOfPyovvrMeO48++ZP45aTsvtOecO8e/3jn/7e+jpnf+/x+IF\nxtF9J2o38k0R0Tsi/iSi1jUkIj4SEd+M2sPovqi9FXvFK63HtqeUFhxh/fenlPZG7U3dZ6I2eX5A\nLZhrKcp/F7WvPy2PiCc6/+hFseySiPh+RKxMtd+E0YXkFJNzbo+It0ctWHBr1MbYJyOiKee8N2pj\n+UcRsTMibouI+7r8u2r8fCdqYUSro/ZbwR8e5fMPRS0gaXpErIraN5e+GRHupdobojY5PxC1bzsd\n6PwcnGK609jNOR/MOW965Z+o/Ve7js7/fyhqmUo/jNrPRlZG7dsaN+acXy5/ZNCddKdx3MUtEbEr\nOr9m3UW/qLWX3Bm1Z563RsTbcs7bj7QN6H664bg92rPDNzpr747ac/aBiHhf55+NjFr7yb1R+/lA\nR0T81pG2D91DDxvHV3Su69qI2JX+rTtK18yOAxHxyrfmloQPFO8RUs780sBJKT0StW9DfPNkb4uS\nUjonav/173R+qwcAAAAA6Mn4BkY3k1L6rZTS6SmlARHxXyLifl5eAAAAAAB6Ol5gdD+/H7XexSsi\n4lBE/MeTuzkAAAAAAJx4/IQEAAAAAABUHt/AAAAAAAAAlccLDAAAAAAAUHmvKbNwSonfm+Ckyjmn\n413HmWeemVtaWgr1pqbi+zz3EytXP3TokKyXWfdpp512wtbNdhc1Yrsd9Znbt2+PvXv3Hvc47tOn\nTz777CN1Q/w3KemPc3V3rDo6Cq3Ho7m5udS6HbXusutw2112PYraPqdRx/vV3m43ttW1sGfPnti/\nf/8Jm4/VMTkZx68RGjUeysw9jbi2q6TMvaHsvre3t2/LOQ8+9q2r6devX93PFi+/rLso9+rVS9bd\n8mpf3bl09zq3PNtddCK3291LFTUnN/LZ4qyzzirU1f64fSk79zbinL3wwguyrvblwAHdYdSds4MH\nda8EtS1uu1/zGv1X7hdffFHW1fOd28fevXvL+u7du+tet9sOd98pE0Hh1qHGz86dO2Pfvn1HHUCl\nXmAAPUFLS0t86lOfKtTPOOOMQs1doG7y27Nnj6yfeeaZhZqbLIYOHSrre/fulXW2u+jV3m734PEv\n//Ivhdodd9whly3r7LPPjlmzZtW1rLu5ue12N2C1P/369ZPLugcBdyPbt29f3dtXdrvVw5R7mHD2\n798v6+oG7PbdbXeZv4A3aruV008/XdbVQ9Odd95ZajscNx+rc+keXN3DZZnjXfYhrcxLBnfOXN29\nPHXnR3HXvLqGI/TxdttR9gWBUuZlTISfv9W5d+fdjZOPfvSja0ptjNHS0hKf/exnC3V1P9qwYYNc\nx8iRI2V98+bNsq7GUNl7nZsj2O6iE7nd/fv3L9TctbZz585C7Ytf/KJctqyzzjorbr/99kJ92LBh\nhZp7NnPXt6v36dOnUHPnrLW1Vdbnzp0r69dcc02htnjxYrms2seIiK1bt8r66NGjC7WNGzfKZQcN\nGiTrq1atkvXrrruuUJs3b55cdsKECbI+e/bsutfttsM947300kuyrqjzG6GfLf7+7/++rnXyExIA\nAAAAAFB5vMAAAAAAAACVxwsMAAAAAABQeWRg4JSTUpK/pS7zW/EdO3bIet++fWVdrdv9TtflOrjf\nS6vfkLnfGe7atUvWVa5DhP69tPtdtDt+blvU7+fcOlzd/SZerdtt9/bt22XdHROVjeF+m65+p+p+\nh11Wc3OzDGJSvxt1YZ/Lly+X9VGjRtnPPJwbr+639ipQKyJi06ZNhZr7PWp7e7usjxkzRtZVvobj\nftfpfsOpzmfZ3AQ31tTYLDtvjBgxQtbVb6u3bdsml1XK5B0cbT0uJPRwgwfrnEW3LS4HQq3b5Za4\na2T8+PGyrrIk3O/E3W+lhw8fLutqHLtx6X4/7n6Hre4N7vi5eddd22UCQt315z5TnXuVgRARsX79\n+rq341ip7VFBemq+i/DbXiYEz80Fbp5xGQ5qX1w2hBuH7r6rtmXNGh1ForIhIvxzmLpOtmzZIpd1\nz2zueKtnkbVr18pl3Xzlzo+a7901qM5ZI58t1DhUc5u7XpcsWSLr7h6t1u2uEXf/d3khK1asKNRc\nBob7TDdOygRtrlu3TtbdWPv2t79dqLW1tcllJ06cKOvueWHBggWFmsv5cM9PLotEnR93LagxX+/f\nxfgGBgAAAAAAqDxeYAAAAAAAgMrjBQYAAAAAAKg8XmAAAAAAAIDKI8QTp5yOjg4bqnY4F67jQqXc\n8mWC3QYMGCDrLmhLhfS4UDsXtOWCftQ2unBGF7jo9lN9plt3S0uLrLtjokLTXMCVC7Vz61Zhcu68\nq6Ayt2xZOWe5T2U+0wXGufGjwpVcgKmru0AoFQDowpxcMKXbTxWE5vbRhTmWCYp02+HGlNtPdQzd\nteqCH134obpG3PWn1u22oywXqqzmExcS6ULTXHhkmQDSsmOtzHw8duxYWXdjUAXJufnVBfG6MGh1\nT1P3rSNx+6mOlVvWjVe33eocu7Hp7tuNknOW+9WvX79Czc29LnDZBT+qseLOmwuEdGOozHa7+6gL\nRVTXsns+cfvjAkJVUKS777jj7eaUIUOG1L2sO1YuhFIFA7vwUbUdbt4oK+csrysVSu6uNTdvumOi\nzr07Z9OnT5d1FwKrnonccX3nO98p6w899JCsq3HsguHdnOzuJeoe6EKl3/Oe98j6I488Iuvqnu6O\nycyZM2XdPS+o+5QLWB05cmSh9thjj8llD8c3MAAAAAAAQOXxAgMAAAAAAFQeLzAAAAAAAEDl8QID\nAAAAAABUHi8wAAAAAABA5dGFBKecjo4OmW6sEv/LdDWI8EnhqvuAS1t3SdK7du2SdcVtt6u/9NJL\nsq7Sv912u3TpvXv3yro6Ji7R2u27S7LfuHFjoTZu3Di5rEqojvBJ0ur8uPR0ldLsOiOU1dHRIdev\ntmXbtm1yHT/96U9l3SVxq64nbn9c1xeX+q620Y0pd85cWr1K13bdfsp2J1HXlDsmKsU9wnfXUNel\nW9ZxCfmqq4Mb88uWLat7vcdCHS81l7q5ZMeOHbJetoOI4o6Ju+bV8qqbQ4S/Rlx91apVhdro0aPl\nsm4udfuu5hK3HS593l2XKoHe3Udc5wE3F6hx6JZtVJcGp6OjQ26PGitujli9erWsu/OpnkXcutet\nWyfrbk5RHQ/curdu3Srrbrw9//zzhZq7z7v7l+raEaHnTTfe5s2bJ+sTJkyoe1vcXHjvvffKurtm\n1TORuwbVuXTPcWV1dHTIuVbtp+uk5LpOuPGjrns35l23EXf/V11fpk2bJpd1z/Cu483atWsLNdXh\nK8Lvj3vmmDNnTqHm5ra2tjZZd9S9xN3T3JzsqPvD4MGD5bLHM2b5BgYAAAAAAKg8XmAAAAAAAIDK\n4wUGAAAAAACoPF5gAAAAAACAyuMFBgAAAAAAqDy6kACdhg8fXqi5ROKyCcsqDd8l57vUaZdUPGjQ\noELNpQa7FHaXBKzSsl3a+IEDB2TdHZO+ffsWai7FXqVzR/j9vPDCCws1t4/ueLtzrLpXuH1U+1Om\nA8KR5JzlPrnuGsoNN9wg6y6xXaXVl+14485la2trobZ//365rOOukZaWlkKtUedBJZ+7sePS/l1X\nI9UZySnbYUjNd2PGjJHLqrT/xYsX171tR5JzltegOiZuH0eNGiXrbnnVFcR1QBg5cqSsu2tepa27\nedcl27t1T506te51u2vBJfWr411m2Qjd2eZI26L0799f1t25HDp0aN2f16gOUEeirn3VOWDIkCHy\n33f3I3ds1f3fPVu4e0OZzmLu3LsOC+4era4rt33uGnTrVteVG1duu93yak52HRbc9ePuawMHDizU\n3HlX+96oLjvNzc3yXqWOieuK4e5d7v6vrht1j4rwXZBctyc3zyru2lHPEG7dbjvc/d/tj7qvXXTR\nRXJZN07c/V/NG+5acOt2263W7eZe96xUD76BAQAAAAAAKo8XGAAAAAAAoPJ4gQEAAAAAACqPFxgA\nAAAAAKDyeIEBAAAAAAAqjy4kOOW4hGWVGL179265DtedxCV0q+Rll1Dt1l0m1blsRxCXfL53795C\nzSU6uxR/54UXXijU3PFTy0boTiYROuXbpV+7RHC3bpXq7I6rOiZlErGPJKUk06F37txZqPXp00eu\nY9OmTbLuEv/V/rhkbXdMXOK9utZcgvb27dtl3aXYq7Rsd0zc+SnTYcGN47KdT9SYdUnh7vo7++yz\n6163O5dz584t1Mp2iGkEN0+V7S6hxlrZc+Y+U83rLq3dHUOXYl+m483atWtl3XVsUePBXQuu84BT\n5vpz49gtr+5RZTsgNEqvXr1kx4wlS5YUam4+cc8FqpNZhJ8Ly3DPHOrYuvPgnpWGDRsm69u2bSvU\n3D3XjWU3X23evLlQc/Omq7vzoLp/uHvali1bZN1dg6tXry7UXIcTtY9um8tqamqS50Ltuzs3auxE\n+O47at1uHnTXt9t/VXfn3T07u3lWzW1uH1V3jgh/L1H7P2/ePLnspEmTZN3dv9R2u/n+iSeekPVx\n48bJuppnVSezCP2MV2/XKr6BAQAAAAAAKo8XGAAAAAAAoPJ4gQEAAAAAACqPFxgAAAAAAKDyCPHE\nKSfnLEN9VJCOC19zIUwuAEiF8agQpggf9OMCyFQYkQsocsFhLhRJBQCqsKWI8gGFp512WqHmwnvc\ndrsQPBXu5YKSXBCYW7faRndMVKikOzfHQm2LGtsuGG/gwIGyrs5NRMiAOjd2XPCqC4pSAY1Dhw6V\ny/7yl7+U9f79+8u6CsB75pln5LJurLmA0DIBkm4+ceFjZ5xxRqHmjrfjwsfU+HYBfZMnTy7Unn76\n6VLb4aSU5NyhrhN3nNw15caa20/FBYe6+U7NBe4clAmGjYhoaWkp1Nx9xAXgunA9NR7c/cwFNrpr\nfuLEiYWa23d3Dbvl1fXn9l1dT42Uc5bjRd3T3dh084wL2lRztQuVdOfTjWV1b1ThmxH+2nTLq7Hc\n3t4ul3V27Ngh6+qYuDnC3QPdusuMIRdC6a4TNS+5z1Pb7a6Rspqbm+X61dh2c5sLZHXnQe2ne25x\n63bhmSpEd+PGjXLZc845R9ZdyLM6Z+5affLJJ2XdnTc1V69atUou6wI13b1EzT9uHpg2bZqsu+cz\nNSbcmFfXqhtTh+MbGAAAAAAAoPJ4gQEAAAAAACqPFxgAAAAAAKDyeIEBAAAAAAAqjxcYAAAAAACg\n8k6JLiSjR4+WdZWo3yhTpkwp1J5//vlS67j99ttlfdasWYXa9773PbmsS8P9/Oc/X2r5niSlJJP8\nVSqvSwd2x8ml5z744IOFmkvtdinsbt1qPS6FvUz6dYTumuC226V2q7TxiPo7aLhlI3wHEcWlvpft\nqqK2xS2rjneZrhVHklKS41Ptp0ufd9vijtVjjz1W9zrGjx8v6+6cqbHmUuldOrnryKD233UycWne\nLuF8+PDhhdqGDRvksq2trbJ+zz33yLqaCyZMmCCXdXOVG5sqUd513FDzQ9muQ0ei5jbVdcnNa44b\naytXrqzr8yL8WBszZoysn3nmmYXaunXr5LKuW4RLvFfLL1q0SC7rjtXUqVNlfcaMGYWae2ZxnTLm\nz58v688++2yhdtFFF8ll3XzsjpWaN9y5PNFcFxI1/7g51nUqUh3BIvSziLuOXQcD1/lMHXPXvcE9\nE7n5Sl2b9913n1zWPRPdcsstsn7xxRcXanPnzpXLumcfd82qDhiua5e7Bt28pMbt9u3b5bKqQ4Ub\nU2W9/PLLsX79+kJdddFQHYaOtC2u64u6B15++eVyWfdc4O676hy7Llpunhk8eLCsq+v14Ycflsu6\nznZ/8Rd/Ieuqi43rZFL2mlfj2D2zle00p+Z718lEjRP3vH84voEBAAAAAAAqjxcYAAAAAACg8niB\nAQAAAAAAKo8XGAAAAAAAoPJ4gQEAAAAAACqv8l1ImpubC7Xf+73fk8t+5CMfkfWhQ4fKukuVrQqX\n7q+Sin/3d3+31LpdavIf//Efl1pPd6WOrUoTVonyET7x3yVXq+Tum2++WS7r0u1dwrLaFpfa7RKW\nXWK0SjZ215M7Ji7BWC3v0uNddxI1P0ToJHuX7H/GGWfIutsWtZ4yXVzccSrLJd6rse3OQZlk/wg9\njl2XJ9e1w9VVOrnb7iuuuELWXXcENR7OO+88ueyIESNk3Z1jlb7vUrRdyvd1110n66rDhOv+4ca3\nu4+oceiuJ1dvhJxz3en5bh+d1atXy7rqSPDWt75VLuvmO5e+rzogbNu2TS7rOkts2bJF1tX17rqp\nuWvbdQHYtGlToXb++efLZSdNmiTrblt+9KMfFWruGnHXsFte7WeZzjuNlHOWnSfUPcaNeXd/UMn+\nEXoMufPj1j1s2DBZV+t53/veJ5d1nTVcByzFXWtuXLn9VNeP2z7XKcTdB9RYvuCCC+Sy7l7njon6\nO4nrPlema16jqI4y7tnMPRf84z/+o6yruVp1XIzwnYpmzpwp669//esLtXe84x1yWTfPuLlDbcvr\nXvc6uey5554r625OVt2yXHdKtWyE7wKkumu57XDPwu7aee1rX1uoueOnnk/cmD8c38AAAAAAAACV\nxwsMAAAAAABQebzAAAAAAAAAlccLDAAAAAAAUHm8wAAAAAAAAJVX+S4kquPI1772tVLrcAmqDz/8\ncN3rWLZsmaw/8sgjsq4SbltaWuSybW1tsn7PPffI+hNPPFGo3XHHHXLZj33sY7Lu0mZPFSppWCU4\nqw4DEf74ue4uV155ZaHmErRdCrJL4FcdI9x2vPDCC6XqY8eOLdRcmrDrXOHS1lesWFGoufR4l+bt\nkvZVsvH27dvlsu667Nu3r6yr1GmX/r1z585C7UQn4atUcDded+zYIeuuW8ZnPvOZQk11XYjw85rr\npqNS9t15d8fwe9/7nqy//e1vL9TcuHzmmWdk/dFHH5V1lXjvUsjddelS9u+9995CzSWZqw4xET7Z\nX22L65ahuHmqrJSS7HKizrGbH9yYctt42WWXFWof/vCH5bKu+0y/fv1k/atf/Wqh5jqWrFmzRtZd\nUr+6n7e3t8tlnZ/97GeyruaqcePGyWXdWFP3i4iIQYMGFWruuD722GOyPn36dFlX59h1d1m/fr2s\nN4rrQqKuNfds4brjuU4hkydPLtRc9zR1HiJ8J4nvfve7hZqb79098Atf+IKsf/3rXy/UXNcgN1Y+\n/vGPy/pPfvKTQk11RoiIuPHGG2XddcNRz0quK4brBufOvXpGcZ241N9rGvlsoa6r/fv3F2ruudRd\na65jy6c+9alC7dZbb5XLum5/7v5w1VVXFWq33XabXFZ1Y4qIuPvuu2Vddb9058HN9+o+H6HHz9VX\nXy2XdZ07XJc9dQzd86C7LqdOnSrr6u8C7n55PPgGBgAAAAAAqDxeYAAAAAAAgMrjBQYAAAAAAKg8\nXmAAAAAAAIDKq3yI55w5cwq1D3zgA3JZF5CycuXKUss3wve///0Ttm4V0DRz5sxS6/jhD3/YqM3p\nllTgjQqSc+FWatkIH9qkQiWXLl0ql3WBmi5UUu1L2e12QT+bN2+uex0uME+FHEbowKorrrhCLusC\nwlzokAq4UucgwoccqlC7CL3/LjxKHVe37LFQ517ND27fXfCqC+lTwU+LFi2Sy7r51QVzqnW7cEYX\ndOeO7S9+8YtCzYVeudA0d6zUOHGhc267n3zySVlX15S7/oYPHy7rbn92795dqLl9VOt221FWzlkG\nqpa5Tty+uzBHNZf+/Oc/l8u6ADO3ffPnzy/UVPB2RMR5550n6+4aUetxAYIunNiNTRU+6cIg3Vzv\nAnBVgJub011IrdtuNXbK3p8bJecs773qc929y12vLrBaPS/86le/ksuqwM8IHc4YEbFq1apCzQXX\n33DDDbLuxrgKLnQBri5w8emnn5Z19Wz1R3/0R3JZd/939y8VuOyet9z14wIuW1tbC7WtW7fKZdUc\n4UKij4W6P6qQXhd6OXr0aFl3z7EqDFQ1Q4jwgbYuUFSNE3cvduPVhb2qa8edXxfu6earMvP9jBkz\nZH3KlCmyruZkN8e6Y+KeoRT3DKqunXoDwvkGBgAAAAAAqDxeYAAAAAAAgMrjBQYAAAAAAKg8XmAA\nAAAAAIAytc42AAAgAElEQVTK4wUGAAAAAACovMp3IVmxYkVdtVPJWWedVahdcsklclmXLu2S1U8F\nHR0dskvFrl276l6HS512Kf4qcd11uVDnN8InTKsU9j179shlXWqw60KijpOj0rkjIp577jlZV8fq\nnHPOkcu2t7fXvY4Ifaxcurs7Vm7fVZqySxtX2+fSmMvKOcvOAQMGDCjUVMeXiIhf//rXsn7NNdfI\nupo3XLL/wIEDZd11H1Cp4O5Yuc42Lll8yZIlhZobayNGjJB1d/2pBPGbbrpJLqvOV4S/LtVnunnD\nHe+9e/fKuro3uHGybNmyQs11SzoW6jyr/XSdNdzxc9f28uXLCzXVPSQiYvbs2aU+801velOh5rqK\nuHPj7gHqmLguCm773PWqjtWYMWPksm7edWNQ3Vtd0ry6nx3pM1U3nVGjRsllG9mlQeno6IgDBw4U\n6upac/vvrkF3/9q2bVtdnxehu4pE+HGoxq3rauCO7YQJE2RdzSnuPuru/64zy6233lqoXXXVVXJZ\nd4/5zW9+I+uqw4R7VlDdryJ8dyS1Hte5QnXtcveXsl588UV5ftQcoTqnRPhx7LqqfOMb3yjU3Dlw\n3ercvfu9731voebGq6u7Z341vt29xJ0f1+VDzadvf/vb5bLO888/L+tq/nH39IkTJ8q660Cj9lN1\nPXHbUS++gQEAAAAAACqPFxgAAAAAAKDyeIEBAAAAAAAqjxcYAAAAAACg8niBAQAAAAAAKq/yXUhO\nZa4zxN/+7d/WvY53v/vdsv70008f0zb1FCr51iXcKy792iUV9+3bt1Bzib8uoXrfvn2yrlKTm5r0\nu0mXzu46GwwaNKhQc+n28+bNk3WXVKzSlDdu3CiXdZ2HXDK76gzh0sZd6r2rqy4fZbq7uHPTKGps\nu2TtSy+9VNbdsVLjx+27O36uC4m6/twcqJL+I3yitVq3OyZu391+qg5QrqPRxz72MVm/8sorZf3i\niy8u1FwXid69e8u6m2eGDRtWqLlzM378+EJt8eLFctmympqa5HlWHQncPrpz6eYe1V3h+uuvl8u+\n/vWvl3U31lQqveuGou4LEb6rgZo7XMcJt31u/jn//PMLNdXhI8Lfi9x8rNL01RwdETF8+HBZd8dE\n3S/d3NOoDlBOU1OTPKdl7qOuU4HbdnXNTp8+XS7ruo245wK1bveM49bt5vCrr766UHP7uGjRIll3\n3X2uvfbaQs11Q3MdI1xHK3WduO477li5/Sxzn1LXYKO67PTp00c+G6jPdPvo5h93ztR1P3XqVLms\nm2dcXXV1cs+UrsuO656ycuXKQk2N7YiIZ599VtbdNXLuuecWaq67i5s3XPcm1flEdbY50rrdvUQd\nb/XcHKGflep9RuYbGAAAAAAAoPJ4gQEAAAAAACqPFxgAAAAAAKDyeIEBAAAAAAAqjxcYAAAAAACg\n8uhCUmFveMMbZP23f/u3617H448/3qjN6TFSSjKt2SXtKi6d3aX4qyTgPn36yGVdQrVLbVedO9x2\nuA4Gy5cvl3WV/O4S9ZcuXSrrt912m6yrtH6VXhwRcffdd8v6mDFjZF2tx3U4cefBpeGrhHt3zlTC\nskvIPxYqrfngwYOFmkrBj4h44IEHZH3mzJmyrjoeuO49mzdvlnXXeUF19nHdRlzauju26ji5Tgpr\n1qyR9csvv1zWVbeMG2+8US570003ybrrBqMSzt1Yc2njI0eOlHW1Hnf9zZ07t1Bz3S/K6ujokNuu\n5h6XbO+2xXUHUNelOo8RPhHdzT2qG4O6JiP8nOnS9FUSvppHI3yXC7efihtrrrONO1bqunT3kS1b\ntsi6686ljm3Z7i6N0tzcHP369SvU1VzoznFbW5usz5gxQ9bV8XLn3l3fbo5YsmRJoeY6JrguJKrb\nUYSew92YcM8ns2bNknV1bN2zjxtXbrvVWHZzr3umVGMkImL16tWFmuuGosaU6whS1qFDh2LHjh2F\nuuok4bq7TJw4Udbdc4E6ru4ZTB2nCN/hRJ0Hd24WLlwo665z49q1awu1d73rXXLZe++9V9a/9KUv\nybrqlOI6cbnOZ+5Ybdu2rVBz58aNK9WBLULPBWo8ue1w19Ph+AYGAAAAAACoPF5gAAAAAACAyuMF\nBgAAAAAAqDxeYAAAAAAAgMojxLPCXOiQ8rnPfU7WXXDKqSylZMPG1LKKC2FywYAqKGrTpk1yWRfA\n6QLIxo4dW6ipsLcIHybngoFUCJ4KWzxSfdmyZbKujmF7e3up7XMBXDt37izUXGjT9OnT615HRLlA\nSBV65cJYj4Uan2WCV10wpTveat/dNeJC0FwInFqPO2cu5Mkd2zKBlZMmTZJ1Fy6ojvc//MM/yGWv\nvvpqWR8xYoSsq2Po9t2FM7qxWSb8cPLkyYWaCzUrK6Uk5w41D7p9d0Gyjgp7c2F5LhjWBYqqMDUX\nOuu2210j6pmgbKiieyYoE+5Z5tkkQl9/7l7kgvjKBPe6c+aOVaMcPHhQnn81n7pj+Ja3vEXWXSCt\nunbcut09bcOGDbKurkF3DF0QqHsuUPOSCwt0c7ILU1fPFm4edPcBNxeqbXQBii4g3D3LqfBRd7zV\neHDPjsdCrUsdQzcnu/PuAsVdMLDirgU3h6nnAnfPdXPHFVdcIevqGcqFj44bN07Wn332WVlXz49u\nXnfz6QUXXCDragy6+5HbHxeyXmYcqmPiAo4PxzcwAAAAAABA5fECAwAAAAAAVB4vMAAAAAAAQOXx\nAgMAAAAAAFQeLzAAAAAAAEDl0YWkAlxa7yc+8QlZ3759e6H2ta99TS6r0r9PdR0dHbF///5CXSVJ\nu4Rql/jr0nPVOXPLuoRql9C9bt06WVcWLlwo6y5RX233Qw89JJedPXu2rLsOOSqh+6677pLLuq4x\nixcvlnXVheXiiy+Wy7q0cZcsrlKq3fapZO1GXZMpJZmurVLBXZK5S5F2nUXUOHEp5GWpLh8u/dpd\nC+7YqjnWLes6+LhuNSrN/Ac/+IFcdvPmzbK+ZMkSWVfdBFyHATdXuf0s0xlCXav1dnI6mo6ODnmd\nqA407l7putW45dW17boUuPEwZcoUWV+wYEHdy7quS9OmTZN11UFEJdVHRKxdu1bWzzvvPFlXCf5L\nly6VyzquU4g6P65Dg0uwd+dH3cvd/HCiNTc3y+tQzZErVqyQ63AdFtx9XtVHjRoll3344Ydl/eab\nb5Z1dV26ZwX33OLmH3Ufdfcp122sTNeptrY2uaw73m6uVnOHu48OHTpU1t3YVx0m1DOYW9Z10Cgr\n5yyvWXWtuev1gQcekPULL7xQ1tW93p1f1yXNnQd1rNy9wY1j95yjjsny5cvlsm4+nTVrVt2f6TqW\nuDnPjTU1V7u/k7jnAjcnq+dHd52p66neZ2S+gQEAAAAAACqPFxgAAAAAAKDyeIEBAAAAAAAqjxcY\nAAAAAACg8niBAQAAAAAAKo8uJBXwh3/4h7J+ySWXyPqPf/zjQk2lk6Oc0aNHF2quG0OZFOkIne7r\nUvxdyrdL/B02bFih5tLgXZKy68ShEs4ffPBBueyXv/xlWb/ssstk/c477yzUXIr2kCFDZN11H5gx\nY0ahtnv3brmsSzwuk4Ddr18/uaw63i61+1ioMeTGj+I6GDgu/Vtxx9WlfKsuCO78uvHq0rLVudyz\nZ49c1iXHu+O6cePGQs2Nhze/+c2y7q5XlcrvlnXcNa/GoRvzrt4ITU1N8nyqWpnuOEeqq/nbXZcT\nJkyQdTfW1H1bje0I31HGpcGre9SiRYvksq4ThdvuVatWybrS0tIi6+78qM90XQDceXBjUF0Pblm3\n742k5j2VwL9hwwb575d9thg/fnyh5p4V3vGOd8i6M3LkyELNzSduzLr7QL3dWiL8nFymu5br0qS6\nSEX454XBgwcXau7cuOvb7afaFvecqJ6VGvlsoc6b2h83t7nOS62trXVvg+s+o+65Eb5TT5l7iZuX\n3LOS6hIzb948uextt90m62PGjJF1tR43plwHFsetRyk7V6vr1XVJUet25+ZwfAMDAAAAAABUHi8w\nAAAAAABA5fECAwAAAAAAVB4vMAAAAAAAQOXxAgMAAAAAAFQeXUheRS5N+NZbb5V1l6B+xx13NGyb\nTkVNTU0y6Vul5LquC5s3b5Z1l5qsUqddUvrOnTtl3aWTv/DCC4WaS/x19ba2NllXKcgf+tCH5LLn\nn3++rC9YsEDWV65cWai57hIusV0lgkfoVHDXvWHbtm2y7s6PWrcbJ+pcNioJP6Uk5xSVHr5p0ya5\nDrfdLllcdTtyy7r9VIn8ETpV3aW1l0m0jojYv39/oeaSrl2y9vTp02X9W9/6VqHmEvndeHXdd3bt\n2lWoufuC69jiPlNx14JSptvN0agxq+Y1Nwe6OdN1HlCft2XLlrqXjYiYOnWqrKvuNu7e77bbJcSr\ncTxo0CC5rLvO3PhWY1B1uIrwHRrc/K2uVze/tre3y7rrfFJmO9asWVP3Oo7FoUOH5PlX5809Q7gx\n7saQmiPWr18vl3Xzo7sPqDHhuje5jiDO8uXLCzU3b7rzOW7cOFlXnSFcpz53/bj5VN3X3POJuze6\ne4/qrlHmvLv7Ylk5ZznvqX1342HJkiWyvnTpUlmfP39+oeauBXcu3d+l1PXnroWtW7fKuuuEo65j\n1QUvwo9jN5+qY6g6A0X4Mei2W40rdZwi/LOcukdH6O1290v1me556HB8AwMAAAAAAFQeLzAAAAAA\nAEDl8QIDAAAAAABUHi8wAAAAAABA5RHi+Sr69Kc/Leuve93rZH3OnDmy/tRTTzVsm05FOWcZEqNq\nLizIhQu58BkV2uTCmdy6XcCVCtNz63bhcG4/VZDVBz/4Qbms2243XlXQmAv6GTFihKy70Cp1TFxA\nkQu7cwGS6jy4462CzdyyZaWUZDDpgQMHCjUX3OdCDl988cW6l3eBZO74udAv9ZluTLlgRRcsqZZ3\n14IL4l23bp2sP/HEE4WaC7R1x9sFh6ltKTPWIsqdB7fs8OHDCzUXwngs1BhS++PmQHcu3dhUXJif\nu3ZcgJm6Jl1gXL9+/WTd7afiQtrcuXSBvmoedGPN7Y/7THUM3T66688FTap7gDtnZQJtj0XOWc5j\nKmzSzYPufqTCoyP0Prn5e/z48bKuArUj9Ph091F33tT9KCJiwIABdW+fmzddEKo6Vu7eMGTIEFl3\n870Ktt23b59c1j23qLDOCB366eYZdd27oPKympqa5DWurm933t215q77SZMmFWruudSFo7rnQTWO\n3f2rtbVV1idMmCDrav/HjBkjl3V/P1ixYoWsq3HijvfkyZNl3c2F6pi4ecNdw+75TF3H7t6g7rtu\nmw/HNzAAAAAAAEDl8QIDAAAAAABUHi8wAAAAAABA5fECAwAAAAAAVB4vMAAAAAAAQOXRheRVpFJ2\nj6Stre0EbQkUlZLr0q9dffv27bKuEo9dKrZLtHZdBtTyLrnapcq7BH6VOL5gwQK5bNmuBGPHji3U\nXKq4S+t33QdUarJKPT/Sul0CttpPl8asxpTroFFWR0eHPPcqwdl95qJFi2R92rRpsl4m8d91gFCd\nbSIiDh48WKi5RH63P27cq+PktqPsWFOJ9y6pf/78+bLurj+VCu6W3blzp6y71HeVWO/mr2XLlhVq\n7liXlXOW514d1759+8p1lJ0z1flx88O2bdtkvaWlRdbVNeK2z3Fjc8uWLYWau484bq5Sc6/bx4UL\nF8r6xIkTZV2NlSlTpshlXYcBN8+ocdKI7i7HoqOjQ957VM11JHj22Wdl3XWYUGPLdRO4//77Zd3N\n9+r+5cabm3/cHK46n7g5Rc0PERG7du2S9dWrVxdqrnuDWjbC76fabne83XzvuvioMeHGrOpc5eaN\nsg4ePCjvBapzjnsWdt081q5dK+vqmnXn3d1f3RyhlnfnwD3Hus4xP/jBDwq197///XVvR4S/r9V7\nX4zw15/rUqXW4+Ze10HEnR/V2cfdd3bv3l2o1XtP4xsYAAAAAACg8niBAQAAAAAAKo8XGAAAAAAA\noPJ4gQEAAAAAACqPFxgAAAAAAKDy6EJygqg03De+8Y1yWZfket999zV0m1CTc5ZJ8arLgEugd2m9\nLj1XJTW7xGjX7cCllqtuDy6N2W333XffLesqTfhtb3ubXPapp56S9SVLlsi6SiV2acfjxo2TdZe8\nrDo1uA4LrtuIo86PO++qC4LrBFNWR0eHHEOqm4IbO+64uv1R2+72x3UCcMurVHXVdSHCp3a7FHuV\nLO66Trh08n/+53+W9ZtvvrlQc/vuumK4Y9K7d+9CTXU0iPBp9W6eGT58eKE2ZswYuez48eMLtcWL\nF8tly0opyW1X++66zwwbNqzUZ6r52HV5KJvMrsZa2c5D6v4Uoecet92OmwtcxxHlyiuvlHWXNK+u\nqbIdQdw8reZ6t4/uuDZKc3OzPEdqDLm5113f7p6uupC48+Dqbt5U++KOoZs3VdeOiIhNmzYVahdf\nfLFc1s33rmvZ1KlTCzU397oOIm5/FHcvKdt1Qj1vuflbdQQps81H0qtXrxg9enShru5rajsiIgYP\nHizr7jpWx1DdAyL8GHT3UTXXuM5Q7l5S5jlW3VsjfAcR19VJdRCZMGGCXHbEiBGy7sagGldl5gG3\njgj9HOHmL9dJqB58AwMAAAAAAFQeLzAAAAAAAEDl8QIDAAAAAABUHi8wAAAAAABA5fECAwAAAAAA\nVB5dSE6QmTNnFmqtra1y2UceeUTWH3300UZuEjqllGRyu0oIdgn0LlXedW9Q6b4ufdclSQ8cOFDW\nVRK3W4fbbrdulcjsUsVdmve6detk/dJLLy3UNmzYUGrdLtlYJbm7BGi3DpeSr7oPqE4wEfocu4T8\nspqamuR5VmNbdV2IiJg7d66sT58+XdbVMXQdN8qOQXW8Xcq8S313qeXq+nMdbFxavUvoVonb69ev\nl8u6cezGhErqP/fcc0utw50HtS1lxolLNy+ro6NDrkvNparDR4Q/rmeffbasq+5S7ji51HyVEH+k\nbVFcar5LwlfzibvnuPFQtq6sWLFC1lVHkIiI1atXF2quE4O7X8yYMUPWVRK+W/er0YVEXUMbN24s\n1Nw2uvuU6goRocetuzbdsVXdLyL0M+ikSZPksq4rj5s3VRcW1+1g9+7dpdat5gn3LOeeOdx+Pv/8\n84Wae35yJk6cKOuqE5dbt+qK4c5BWS+//LK896rr2H2muwe6OW/27NmFmjsH7py55xY1t7ntds/l\n7l6ixqx7HnT77q5XNWbdsgsWLKh7+9zy6pk8wnccc3OYGifuOXHNmjWFmusQczi+gQEAAAAAACqP\nFxgAAAAAAKDyeIEBAAAAAAAqjxcYAAAAAACg8gjxPEG+8pWv1L3sXXfddQK3BIoKElThWy70S4VE\nRugAxQgdUOjC3oYPHy7rLnRIhQu54B4XvuZC4971rncVaiqQLMIHbblgRRc+p7j9USGHEToEyIXU\nueA5F+akAppc6JAKm3RBhMdCbYsaJyroLiJi7Nixsu7CptT4dvvjwjodte6RI0fKZQ8ePCjrLtRN\nBS668eACwtz+qBBYFzLqts8Fxqr9dwFhjltehYH16dNHLjt58uRC7emnny61HU5zc7MMA1PH283H\nLnjVnWM117tlR4wYIesu/FeN47LnzAWHqkA/N0+5+dVtS5l5adSoUbLu5vrzzz+/UHNhdO6ad+tW\nY8JdZy7IsVEOHTokP1uN78GDB9t1KC7MUYUlumPb0tIi6y6MT4UounPvApfdeXvLW95SqG3fvl0u\nu2zZMll3262eF9wzxAUXXCDr7vyoucZday7o1wWhq3u32w41HtycVFavXr1kOLUKjH3qqafkOtS/\nH+GfE1U4tZuTxo8fL+vuPKg52Y1j92zv5tlrr722UHvyySflsu4adg0e1HOYu7bdM7z7O4kKPHXP\nfS5M1YV+K7/5zW9k/frrry/UnnnmmbrWyTcwAAAAAABA5fECAwAAAAAAVB4vMAAAAAAAQOXxAgMA\nAAAAAFQeLzAAAAAAAEDl0YXkOLn0eFdXHnzwwUZtDuqQc5ZpxSrN3KUauw4iLjVZJRi7ZV0KsuuC\noLbbjb+FCxeWWvfjjz9eqA0ZMqTu7YjwKciqc4fbDpdO7o6V6iThEtjddjsqJd6lN6tE+rLdOY5E\njaEy3Rvcdrs0c9X9w3UEcWn6LpVeLe/GjjuGrtuKSsgfN26cXLa9vV3W3VhTidmuC4nrsOCOt+oa\n5FLI3bFy84w63u5cqu4kbuyUdejQodi7d2+hrvazzPwa4bsAqC4fbl5z3D2gzPXtEvnLrMOdB9dt\nxK27zPXn1q3OY0TEunXr6vq8CD+++/XrJ+vq3uA6xzSyA5Rbv+oKpuZfl7RfpmtQhO5C4roD7Ny5\nU9Zd9yE1VlyHl/nz58u66/awdu3aQs11FXFjRR3riIg9e/YUaq4DjZvztmzZIuu/+tWvCjXVncNt\nR4TvfKLOgzvvqjOL61RX1qFDh+TcpD7TnTO33a7D0po1awo19xzrOse5blSqc57bbncvcff05557\nrlBz3UZWr14t625sLl26tFBz17br+uKeidS2uGezsl0X1TUyevRouay6ztw1eTi+gQEAAAAAACqP\nFxgAAAAAAKDyeIEBAAAAAAAqjxcYAAAAAACg8niBAQAAAAAAKo8uJMfpvPPOk/Vp06bVvY4pU6bI\n+pIlS45pm3BkTU1NMt1YJfC6hHfX1aFMgrpLVXef6RLeVYJxW1ubXFZ1Y4iImDlzZt11t48qRfpI\nVHL3+PHj6142wneJmTRpUqHmuga4pGuX3qySrt25VKngruNEo6ix6RK3Xd1RnRpcYrQbx255l4Ct\nuCRzl+atErDdvrsOBm7dKiHfjQdXd50HVLJ42U4K7npV63HrPpHdG5qamuQ1pRLO3bXqjqubN1S9\nTOewiPLddxTXEcRdO2q73T66+cvNmWpbXMp82TlMdd9x58wdE7fd6hpx2/1qUOdfdQ5w3TxcpyJ3\n71ZdPtwxdHOe6ggWoec89+zjuvi4jgyqK9jw4cPlsqr7xZGoa9MdP9fJxHX0mDx5cqHmns0uu+wy\nWVdd0iIiWltb61735ZdfXqg99NBDctmyUkryGnfPZ4qbT934Hjx4cKHmxo6bH929Tt1f3Pzt1uE+\nU21j2bnNPVtcdNFFhZrrNuK6k2zdulXWhw0bVqj1799fLuu4das5yXU6UuOs3g5nfAMDAAAAAABU\nHi8wAAAAAABA5fECAwAAAAAAVB4vMAAAAAAAQOXxAgMAAAAAAFQeXUiO06xZs+pe9itf+Yqs33//\n/Q3aGtRLpdyq1GmXKO8S3l06u1qPS/Z36eSOSlNet26dXNZ1enBdSFRHhtmzZ8tlFy5cKOsu+Vyl\nmbv0YXdMXNq8SpJ2+75r165S61bb7RLLy4ypslJKsnuFGoPu+LnuFy5BXB1Xd5y2b98u6yp9PkJf\nD5s2bZLLusR710FEdWpw1/CWLVtk3e2nqrvj7fZnwIABsr506dJCTXVUifDp6e54q3HijokaJ27s\nHAvXYeJw7tpxdTefqM9z17A7ly45XimTjh/hz7Ga68t2xHKfqcaD607h7nNujq33/Eb48+DuI2re\ncOfdXduNpPZ19+7dhZpL8Hfjyu2T6q7luh24seI62SjuWhs6dKisq+4SEbr7x4IFC+SyrsOZ61qy\nc+fOQs11IXF1tz/qecGNWddtxFm/fn2h5saJOu+us0ZZOWe5LjXnu2cFd827bhlq3Lvj6sa3m39U\nd41f//rXclk3915yySWyrjqiuWv14YcflnX3bKG6mbjn1cWLF8u6eyZS15/reOM6iJxzzjmyrrqQ\nuOtMnUu6kAAAAAAAgB6DFxgAAAAAAKDyeIEBAAAAAAAqjxcYAAAAAACg8gjxPE4rVqyQdRVgc/fd\nd8tlXSANTpx6Q8VcmIwLFHPnUgVfuTAsF6jl1q1CzC677DK57DXXXCPrLkRIhVCp8KgIf0wmTJgg\n6yqgaM+ePXJZF3jq6iqA6sCBA3JZFWYU4ceICkIrEypVb0DR0eSc5Taq8+NC91xoowtzKrMOF7Tp\nxpral9bWVrmsC5Vy51gtv3btWrmsCxlzoYhqrLlzrMZ8hA9LVNeOG2tlx7EKp3TXk1pHmWDGY9GI\nwDgXAldv+G2EP2duHKt53QUCuuPtrim1bjfmywRKR+gx60Lx3Bh0263W4+5zbh2uruZjF4Tp7lEn\nmvpcN8+48VYmiNmdYxe8W+ae7tbtrgcXXDxp0qRCzc3Jw4YNk3UXSqpCG93xLhtGPGjQoEKtzLwZ\n4eclFbDrwrCnT59eqLnrtaycszzP6rp3966ygaLq2c/NES7Y1J1LNXe4cFl3725vb5f1MWPGFGqP\nPPKIXNYFhDrq2Lp9dOHMbn9WrlxZqLnx48KPBw4cKOtqDnPjRI2pescO38AAAAAAAACVxwsMAAAA\nAABQebzAAAAAAAAAlccLDAAAAAAAUHm8wAAAAAAAAJVHF5Lj9NWvfrVUHdWg0qHrTV0+Ur1MGnWZ\nDhoRPk1YJXG7RHCXJjxnzhxZHzBgQKHm0ptd3SVxu+TzMlwis6qrfYnw6c0u9V2dH5eGrtLGG9l1\nSG1LmQ4sbltcurS6Rsomn7ttUcfKdZ3YuHGjrLvxrTp0qA47EX4cuwRx1e3BjYfdu3fLuuuOoNLT\n3ZgvO5+o68+NeXUNl02Zd3LOchyqz3Tb5463Gz+KuxbKzLsR+riUveZ37dol62ped3N92W5H6ni7\nsVb2WKl5w+2j69Dg5pky3XTcuk80Nee5Y1W2E1eZDmeuk4u7TtR5dmPCbbfr1KeukzL7GOE7G6j1\nNOJ6cNyzjFuH666hrgl3ramOLe78Noqa89w9t0zXoAh9T3fdxjZv3izrbvyojmiqm0yE74Tzmc98\nRtZvueWWQs11vJk4caKsO2o8qO4hEb5znPvMtra2Qs11y3ryySdl/c1vfrOsq+tyxIgRcll1ztz1\nfohjNdIAAADESURBVDi+gQEAAAAAACqPFxgAAAAAAKDyeIEBAAAAAAAqjxcYAAAAAACg8niBAQAA\nAAAAKi+VSdpNKW2NiDUnbnOAI2rNOesWASUwjnGSMY7REzCO0VMwltETMI7RE9Q1jku9wAAAAAAA\nADgZ+AkJAAAAAACoPF5gAAAAAACAyuMFBgAAAAAAqDxeYAAAAAAAgMrjBQYAAAAAAKg8XmAAAAAA\nAIDK4wUGAAAAAACoPF5gAAAAAACAyuMFBgAAAAAAqLz/D8G8scZlzAXvAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x16481cbb940>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "img_ind = 100\n",
    "\n",
    "# Get indices for 5 feature maps with strongest activations\n",
    "layer = sess.run(activations[6], feed_dict={X: images[None, img_ind]})\n",
    "\n",
    "for i in range(3):\n",
    "    layer = np.sum(layer, axis=0)\n",
    "\n",
    "inds = np.sort(np.argpartition(layer, -4)[-5:])\n",
    "\n",
    "figs = [images[None, img_ind]]\n",
    "\n",
    "for ind in inds:\n",
    "    deconv = tf.gradients(activations[6][:, :, :, None, ind], X)[0]\n",
    "    r = sess.run(deconv, feed_dict={X: images[None, img_ind]})\n",
    "    figs.append(r)\n",
    "\n",
    "fig = plt.figure(figsize=(15,15))\n",
    "for i in range(len(figs)):\n",
    "    ax = fig.add_subplot(1, 6, i + 1)\n",
    "    im = ax.imshow(figs[i].reshape(28,28), cmap='gray')\n",
    "    \n",
    "    if i is 0:\n",
    "        ax.set_title('Input Digit')\n",
    "    else:\n",
    "        ax.set_title('Feature {}'.format(inds[i - 1]))\n",
    "    \n",
    "    ax.set_xticks([])\n",
    "    ax.set_yticks([])\n",
    "\n",
    "plt.tight_layout()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  },
  "latex_envs": {
   "LaTeX_envs_menu_present": true,
   "autoclose": false,
   "autocomplete": true,
   "bibliofile": "biblio.bib",
   "cite_by": "apalike",
   "current_citInitial": 1,
   "eqLabelWithNumbers": true,
   "eqNumInitial": 1,
   "hotkeys": {
    "equation": "Ctrl-E",
    "itemize": "Ctrl-I"
   },
   "labels_anchors": false,
   "latex_user_defs": false,
   "report_style_numbering": false,
   "user_envs_cfg": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}