model_tpu_adapted.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Copyright 2017 The TensorFlow Authors. All Rights Reserved.\n",
    "#\n",
    "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
    "# you may not use this file except in compliance with the License.\n",
    "# You may obtain a copy of the License at\n",
    "#\n",
    "#     http://www.apache.org/licenses/LICENSE-2.0\n",
    "#\n",
    "# Unless required by applicable law or agreed to in writing, software\n",
    "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
    "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
    "# See the License for the specific language governing permissions and\n",
    "# limitations under the License.\n",
    "# ==============================================================================\n",
    "\n",
    "\n",
    "\"\"\"MNIST model training using TPUs.\n",
    "This program demonstrates training of the convolutional neural network model\n",
    "defined in mnist.py on Google Cloud TPUs (https://cloud.google.com/tpu/).\n",
    "If you are not interested in TPUs, you should ignore this file.\n",
    "\"\"\"\n",
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "import os\n",
    "import sys\n",
    "import keras\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "\n",
    "from matplotlib import pyplot as plt \n",
    "%matplotlib inline\n",
    "import glob\n",
    "import cv2\n",
    "from collections import Counter\n",
    "from IPython.core.display import display, HTML\n",
    "import random\n",
    "import numpy as np\n",
    "import tensorflow as tf  # pylint: disable=g-bad-import-order\n",
    "\n",
    "# For open source environment, add grandparent directory for import\n",
    "sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(sys.path[0]))))\n",
    "\n",
    "# from official.mnist import dataset  # pylint: disable=wrong-import-position\n",
    "# from official.mnist import mnist  # pylint: disable=wrong-import-position"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Cloud TPU Cluster Resolver flags\n",
    "tf.flags.DEFINE_string(\n",
    "    \"tpu\", default=None,\n",
    "    help=\"The Cloud TPU to use for training. This should be either the name \"\n",
    "    \"used when creating the Cloud TPU, or a grpc://ip.address.of.tpu:8470 \"\n",
    "    \"url.\")\n",
    "tf.flags.DEFINE_string(\n",
    "    \"tpu_zone\", default=None,\n",
    "    help=\"[Optional] GCE zone where the Cloud TPU is located in. If not \"\n",
    "    \"specified, we will attempt to automatically detect the GCE project from \"\n",
    "    \"metadata.\")\n",
    "tf.flags.DEFINE_string(\n",
    "    \"gcp_project\", default=None,\n",
    "    help=\"[Optional] Project name for the Cloud TPU-enabled project. If not \"\n",
    "    \"specified, we will attempt to automatically detect the GCE project from \"\n",
    "    \"metadata.\")\n",
    "\n",
    "# Model specific parameters\n",
    "tf.flags.DEFINE_string(\"data_dir\", \"\",\n",
    "                       \"Path to directory containing the Omniglot dataset\")\n",
    "tf.flags.DEFINE_string(\"model_dir\", None, \"Estimator model_dir\")\n",
    "tf.flags.DEFINE_integer(\"batch_size\", 128,\n",
    "                        \"Mini-batch size for the training. Note that this \"\n",
    "                        \"is the global batch size and not the per-shard batch.\")\n",
    "tf.flags.DEFINE_integer(\"train_steps\", 200, \"Total number of training steps.\")\n",
    "tf.flags.DEFINE_integer(\"eval_steps\", 60,\n",
    "                        \"Total number of evaluation steps. If `0`, evaluation \"\n",
    "                        \"after training is skipped.\")\n",
    "\n",
    "tf.flags.DEFINE_float(\"learning_rate\", 3e-4, \"Learning rate.\")\n",
    "\n",
    "\n",
    "tf.flags.DEFINE_bool(\"use_tpu\", True, \"Use TPUs rather than plain CPUs\")\n",
    "tf.flags.DEFINE_integer(\"iterations\", 1000,\n",
    "                        \"Number of iterations per TPU training loop.\")\n",
    "tf.flags.DEFINE_integer(\"num_shards\", 8, \"Number of shards (TPU chips).\")\n",
    "\n",
    "FLAGS = tf.flags.FLAGS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "def metric_fn(labels, logits):\n",
    "    accuracy = tf.metrics.accuracy(\n",
    "    labels=labels, predictions = logits)\n",
    "    return {\"accuracy\": accuracy}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "def model_fn(features, labels, mode, params):\n",
    "    \"\"\"model_fn constructs the ML model used to predict handwritten digits.\"\"\"\n",
    "\n",
    "    del params\n",
    "    \n",
    "    if mode == tf.estimator.ModeKeys.PREDICT:\n",
    "        raise RuntimeError(\"mode {} is not supported yet\".format(mode))\n",
    "    image1,image2  = features\n",
    "    \n",
    "    if isinstance(image, dict):\n",
    "        image = features[\"image\"]\n",
    "        \n",
    "    model = create_model()\n",
    "    logits = model([image1,image2], training=(mode == tf.estimator.ModeKeys.TRAIN))\n",
    "    loss = tf.losses.log_loss(labels=labels, predictions=logits)\n",
    "    \n",
    "    if mode == tf.estimator.ModeKeys.TRAIN:\n",
    "        learning_rate = tf.train.exponential_decay(\n",
    "        FLAGS.learning_rate,\n",
    "        tf.train.get_global_step(),\n",
    "        decay_steps=100000,\n",
    "        decay_rate=0.96)\n",
    "        optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)\n",
    "        if FLAGS.use_tpu:\n",
    "            optimizer = tf.contrib.tpu.CrossShardOptimizer(optimizer)\n",
    "        return tf.contrib.tpu.TPUEstimatorSpec(\n",
    "        mode=mode,\n",
    "        loss=loss,\n",
    "        train_op=optimizer.minimize(loss, tf.train.get_global_step()))\n",
    "\n",
    "    if mode == tf.estimator.ModeKeys.EVAL:\n",
    "        return tf.contrib.tpu.TPUEstimatorSpec(\n",
    "        mode=mode, loss=loss, eval_metrics=(metric_fn, [labels, logits]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data_gen = ImageDataGenerator(\n",
    "    samplewise_center=True,\n",
    "    samplewise_std_normalization = True,\n",
    "    rotation_range =10,\n",
    "    width_shift_range = 0.05,\n",
    "    height_shift_range = 0.05,\n",
    "    shear_range = 0.05,\n",
    "    zoom_range = 0.05)\n",
    "    #rescale = 1./255"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "def flip(p):\n",
    "    # bigger p = less TRUE\n",
    "    # True - in same class\n",
    "    return random.random() < p\n",
    "def batch_function(batch_size,features,lengthes,validation):\n",
    "    batch_features = np.zeros((batch_size, 2,105, 105,1))\n",
    "    batch_labels = np.zeros((batch_size,1))\n",
    "    for i in range(batch_size):\n",
    "        if flip(distribution):\n",
    "            not_insame_class = True\n",
    "            while not_insame_class:\n",
    "                indecies= np.random.choice(len(features),2)\n",
    "                not_insame_class = np.argmax(lengthes-indecies[0]>0) != np.argmax(lengthes-indecies[1]>0)\n",
    "            batch_labels[i] = not_insame_class\n",
    "        else:\n",
    "                \n",
    "            first = np.searchsorted(lengthes,np.random.randint(1,len(features)))\n",
    "            indecies = np.zeros(2)\n",
    "            while abs(indecies[0]-indecies[1]) < 20 : indecies = np.random.choice(range(lengthes[first-1],lengthes[first]),2)\n",
    "            batch_labels[i] = True\n",
    "    \n",
    "        image1,image2 = features[indecies[0]].reshape(105,105,1),features[indecies[1]].reshape(105,105,1)\n",
    "        if not validation and flip(0.5): \n",
    "            generator1 , generator2 = train_data_gen.flow(image1[np.newaxis,:],batch_size=1) , train_data_gen.flow(image2[np.newaxis,:],batch_size=1)\n",
    "            image1,image2 = next(generator1),next(generator2)\n",
    "        batch_features[i] = (image1,image2)\n",
    "    return (batch_features,batch_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_dataset(data_train_dir=\"images_background/\", data_valid_dir = \"images_evaluation/\"):\n",
    "    \n",
    "    languages = [path.split(\"/\")[-1] for path in glob.glob(dataset_train_path+\"*\")]\n",
    "    languages_evaluation = [path.split(\"/\")[-1] for path in glob.glob(dataset_evaluation_path+\"*\")]\n",
    "\n",
    "\n",
    "\n",
    "    all_alphabets = {language:[cv2.cvtColor(cv2.imread(image),cv2.COLOR_RGB2GRAY) for image in glob.glob(dataset_train_path+language+\"/*/*\")]\\\n",
    "                                                                  for language in languages }\n",
    "    all_images =  np.vstack([np.array(all_alphabets[key]) for key in all_alphabets])\n",
    "\n",
    "    all_evaluation_alphabets = {language:[cv2.cvtColor(cv2.imread(image),cv2.COLOR_RGB2GRAY) for image in glob.glob(dataset_evaluation_path+language+\"/*/*\")]\\\n",
    "                                                                  for language in languages_evaluation }\n",
    "    all_evaluation_images =  np.vstack([np.array(all_evaluation_alphabets[key]) for key in all_evaluation_alphabets])\n",
    "    \n",
    "    return all_alphabets,all_evaluation_alphabets,all_images,all_evaluation_images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# def eval_input_fn(params):\n",
    "#     batch_size = params[\"batch_size\"]\n",
    "#     data_dir = params[\"data_dir\"]\n",
    "#     ds = dataset.test(data_dir).apply(\n",
    "#       tf.contrib.data.batch_and_drop_remainder(batch_size))\n",
    "#     images, labels = ds.make_one_shot_iterator().get_next()\n",
    "#     return images, labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "def main(argv):\n",
    "    del argv  # Unused.\n",
    "    tf.logging.set_verbosity(tf.logging.INFO)\n",
    "\n",
    "    tpu_cluster_resolver = tf.contrib.cluster_resolver.TPUClusterResolver(\n",
    "      FLAGS.tpu,\n",
    "      zone=FLAGS.tpu_zone,\n",
    "      project=FLAGS.gcp_project\n",
    "    )\n",
    "\n",
    "    run_config = tf.contrib.tpu.RunConfig(\n",
    "      cluster=tpu_cluster_resolver,\n",
    "      model_dir=FLAGS.model_dir,\n",
    "      session_config=tf.ConfigProto(\n",
    "          allow_soft_placement=True, log_device_placement=True),\n",
    "      tpu_config=tf.contrib.tpu.TPUConfig(FLAGS.iterations, FLAGS.num_shards),\n",
    "    )\n",
    "    all_alphabets,all_evaluation_alphabets,all_images,all_evaluation_images = load_dataset()\n",
    "    lengthes_train = np.insert(np.cumsum([len(all_alphabets[key]) for key in all_alphabets]),0,0)\n",
    "    lengthes_val = np.insert(np.cumsum([len(all_evaluation_alphabets[key]) for key in all_evaluation_alphabets]),0,0)\n",
    "    estimator = tf.contrib.tpu.TPUEstimator(\n",
    "      model_fn=model_fn,\n",
    "      use_tpu=FLAGS.use_tpu,\n",
    "      train_batch_size=FLAGS.batch_size,\n",
    "      eval_batch_size=FLAGS.batch_size,\n",
    "      config=run_config)\n",
    "    # TPUEstimator.train *requires* a max_steps argument.\n",
    "    estimator.train(input_fn=batch_function(FLAGS.batch_size,all_images,lengthes_train,False), max_steps=FLAGS.train_steps)\n",
    "    # TPUEstimator.evaluate *requires* a steps argument.\n",
    "    # Note that the number of examples used during evaluation is\n",
    "    # --eval_steps * --batch_size.\n",
    "    # So if you change --batch_size then change --eval_steps too.\n",
    "    if FLAGS.eval_steps:\n",
    "        estimator.evaluate(input_fn=batch_function(FLAGS.batch_size,all_evaluation_images,lengthes_val,True), steps=FLAGS.eval_steps)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "ename": "SyntaxError",
     "evalue": "invalid syntax (<ipython-input-36-f31e864442ea>, line 1)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;36m  File \u001b[0;32m\"<ipython-input-36-f31e864442ea>\"\u001b[0;36m, line \u001b[0;32m1\u001b[0m\n\u001b[0;31m    tf.contrib.tpu.TPUEstimator().train(*)\u001b[0m\n\u001b[0m                                         ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m invalid syntax\n"
     ]
    }
   ],
   "source": [
    "tf.contrib.tpu.TPUEstimator().train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_model():\n",
    "    m = tf.keras.layers\n",
    "    digit_input = tf.keras.Input(shape=(105,105,1))\n",
    "    convolution = partial(m.Conv2D, activation=tf.nn.relu,kernel_regularizer=tf.keras.regularizers.l2(0.005))\n",
    "\n",
    "    x = convolution(64, (10, 10),input_shape=(105,105,1))(digit_input)\n",
    "    x = m.MaxPooling2D()(x)\n",
    "    x = convolution(128, (9, 9))(x)\n",
    "    x = m.MaxPooling2D()(x)\n",
    "    x = convolution(128, (7, 7))(x)\n",
    "    x = m.MaxPooling2D()(x)\n",
    "    x = convolution(256, (6, 6))(x)\n",
    "    out = m.Flatten()(x)\n",
    "\n",
    "    vision_model = tf.keras.models.Model(digit_input,out)\n",
    "\n",
    "    letter1 = tf.keras.Input(shape=(105,105,1))\n",
    "    letter2 = tf.keras.Input(shape=(105,105,1))\n",
    "\n",
    "    out_1 = vision_model(letter1)\n",
    "    out_2 = vision_model(letter2)\n",
    "\n",
    "    def m_dist(A,B):\n",
    "        return tf.keras.backend.sum(tf.keras.backend.abs(A-B),axis=1,keepdims=True)\n",
    "\n",
    "    merged_vector = m.Lambda(lambda x:m_dist(x[0],x[1]), output_shape=lambda inp_shp:(inp_shp[0][0],1))([out_1,out_2])\n",
    "\n",
    "    out_fin = m.Dense(4086,activation=tf.nn.sigmoid,kernel_regularizer=tf.keras.regularizers.l2(0.00015))(merged_vector)\n",
    "    final_output = m.Dense(1,activation=tf.nn.sigmoid)(out_fin)\n",
    "\n",
    "    final_model = tf.keras.models.Model(inputs=[letter1, letter2], outputs=final_output)\n",
    "    return final_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if __name__ == \"__main__\":\n",
    "    tf.app.run()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}