Image_captioning.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Copy of Image_captioning.ipynb",
      "provenance": [],
      "private_outputs": true,
      "collapsed_sections": [],
      "toc_visible": true,
      "machine_shape": "hm",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "TPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/TanyaChutani/ImageCaptionGenerator/blob/master/Image_captioning.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0Ju_sL5i223C",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DYakYlHpp5ji",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!cp '/content/drive/My Drive/Flickr8k_Dataset.zip' '/content/Flickr8k.zip'\n",
        "!cp '/content/drive/My Drive/fasttext.zip' '/content/fasttext.zip'\n",
        "!cp '/content/drive/My Drive/Flickr8k_text.zip' '/content/Flickr8k_text.zip'\n",
        "!unzip '/content/Flickr8k_text.zip'\n",
        "!unzip '/content/Flickr8k.zip' -d '/content/Flickr8k'\n",
        "!unzip '/content/fasttext.zip'"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "K9Dk9PYuIuXp",
        "colab_type": "text"
      },
      "source": [
        "## Importing Libraries"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YdgJqHXw8P-L",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "import nltk\n",
        "import io\n",
        "import cv2\n",
        "import os\n",
        "import tensorflow\n",
        "import string\n",
        "import pickle\n",
        "import tensorflow as tf\n",
        "from glob import glob\n",
        "from collections import Counter\n",
        "import matplotlib.pyplot as plt\n",
        "from nltk.stem import WordNetLemmatizer\n",
        "from wordcloud import WordCloud, STOPWORDS \n",
        "import pickle\n",
        "import itertools\n",
        "import pathlib\n",
        "from tensorflow.python.keras.models import *\n",
        "from tensorflow.python.keras.layers import *\n",
        "from tensorflow.keras.utils import to_categorical\n",
        "from tensorflow.python.keras.callbacks import ModelCheckpoint\n",
        "from tensorflow.python.keras.preprocessing.text import Tokenizer\n",
        "from tensorflow.python.keras.preprocessing.sequence import pad_sequences\n",
        "from tensorflow.python.keras.applications.inception_v3 import InceptionV3,\\\n",
        "preprocess_input\n",
        "\n",
        "nltk.download('wordnet')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "D2oQQ0_h8LuG",
        "colab_type": "text"
      },
      "source": [
        "### Prepare the Photo Feature Extractor"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "P8Z2nrv7m7i5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def pre_process(img_path):\n",
        "  img=cv2.imread(img_path)\n",
        "  dsize=(224,224)\n",
        "  img=cv2.resize(img,dsize,interpolation=cv2.INTER_NEAREST)  \n",
        "  img=np.expand_dims(img,axis=0)\n",
        "  img=preprocess_input(img)\n",
        "  return img\n",
        "\n",
        "def model():\n",
        "  model=InceptionV3(include_top=False,weights='imagenet')\n",
        "  for layer in model.layers:\n",
        "    layer.trainable=False\n",
        "  return model\n",
        "\n",
        "features_df=dict()\n",
        "def feature_extractor(model,directory):\n",
        "  for img in os.listdir(img_root):\n",
        "    img_path=directory+\"/\"+img\n",
        "    preproceesed_img=pre_process(img_path)\n",
        "    feature=model.predict(preproceesed_img)\n",
        "    image_id = (img_path.split('.')[0]).split('/')[4]\n",
        "    features_df.update({image_id:feature})\n",
        "  return features_df\n",
        "\n",
        "img_root = '/content/Flickr8k/Flicker8k_Dataset'\n",
        "inception_model=model()\n",
        "features_df=feature_extractor(inception_model,img_root)\n",
        "output = open('myfile.pkl', 'wb')\n",
        "pickle.dump(features_df, output)\n",
        "output.close()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "m8Gx4-JORklP",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "infile = open('/content/myfile.pkl','rb')\n",
        "features_df = pickle.load(infile)\n",
        "infile.close()\n",
        "\n",
        "features_df['2098418613_85a0c9afea']"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "g5Ib0UtRI0hb",
        "colab_type": "text"
      },
      "source": [
        "## Prepare the text data\n",
        "### Making dictionary of the text file \n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VnABY4J637fg",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "text_df={}\n",
        "directory='/content/Flickr8k.token.txt'\n",
        "doc=open(directory,'r')\n",
        "\n",
        "def make_dict(doc):\n",
        "  for i,line in enumerate(doc):\n",
        "    token=line.split()\n",
        "    img_id,description=token[0],token[1:]\n",
        "    description=' '.join(description)\n",
        "    img_id=img_id.split('.')[0]\n",
        "    if img_id not in text_df:\n",
        "      text_df[img_id]=list()\n",
        "    text_df[img_id].append(description) \n",
        "  return(text_df)\n",
        "text_dict=make_dict(doc)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ftGg4sGDxWSp",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "text_dict.pop('2258277193_586949ec62')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SjCS5RWwoEGq",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "text_dict['2098418613_85a0c9afea']"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Cy2WTFFxY9u6",
        "colab_type": "text"
      },
      "source": [
        "### Preprocessing Of text\n",
        "### Adding start and end sequence"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "51knEZ_tYlF3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def preprocess_text(text_df):\n",
        "  porter = WordNetLemmatizer()\n",
        "  punct = str.maketrans('', '', string.punctuation)\n",
        "  for img_id,descriptions in text_df.items():\n",
        "    pre_processed_words=[]\n",
        "    for description in descriptions:\n",
        "      words=description.split()\n",
        "      words=[word.lower() for word in words]\n",
        "      words =[word.translate(punct) for word in words]\n",
        "      words=[word for word in words if word.isalpha()]\n",
        "      words=[porter.lemmatize(word) for word in words]\n",
        "      words=[word for word in words if len(word)>1]\n",
        "      words='start '+' '.join(words)+' end'\n",
        "      pre_processed_words.append(words)\n",
        "    text_df[img_id]=pre_processed_words\n",
        "  return text_df\n",
        "text_dict=preprocess_text(text_dict)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "99l6JRVCoQ38",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "text_dict['2098418613_85a0c9afea']"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "duWZn3F4JN2d",
        "colab_type": "text"
      },
      "source": [
        "### Creating vocab of words\n",
        "### Checking the most common and least common words\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pDSD-p8We6ai",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def create_vocab(text_df):\n",
        "  vocab=[]\n",
        "  for key in text_df.keys():\n",
        "    vocab.extend(d.split() for d in text_df[key])\n",
        "  words=[]\n",
        "  for v_list in vocab:\n",
        "    for word in v_list:\n",
        "      words.append(word)\n",
        "  return list(set(words))\n",
        "\n",
        "vocab=create_vocab(text_dict)\n",
        "vocab_size=len(vocab)\n",
        "\n",
        "#Checking for the 10 most frequent words in the image description\n",
        "counts = Counter(vocab)\n",
        "print(\"Top 10 most frequent words\",counts.most_common(10))\n",
        "n=10\n",
        "print(\"Top 10 least frequent words\", counts.most_common()\\\n",
        "      [:-n-1:-1])\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yKlYqCw-JaW4",
        "colab_type": "text"
      },
      "source": [
        "### Making Word Cloud of common words\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dO6L0b9a5Lk0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Generating wordcloud with the most frequent words\n",
        "#!pip install wordcloud\n",
        "\n",
        "def most_common(words):\n",
        "  stopwords = set(STOPWORDS)\n",
        "  stopwords.update([\"end\", \"start\"])\n",
        "  counts = Counter(words)\n",
        "  wc = WordCloud(max_words=1000, margin=10, background_color='white',\n",
        "  scale=3, relative_scaling = 0.5, width=500, height=400,\\\n",
        "  stopwords=stopwords,random_state=1).generate(' '.join(words))\n",
        "  plt.figure(figsize=(15,8))\n",
        "  plt.imshow(wc)\n",
        "  plt.axis(\"off\")\n",
        "  plt.show()\n",
        "\n",
        "most_common(vocab)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AbKOkdv_JiUd",
        "colab_type": "text"
      },
      "source": [
        "### Tokenizing words using keras tokenizer"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YwY5RI16gfz8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#creating word to index using keras tokenizer\n",
        "def word_to_index(text_df):\n",
        "  lines = []\n",
        "  for key in text_df.keys():\n",
        "    [lines.append(d) for d in text_df[key]]\n",
        "  tokenizer = Tokenizer()\n",
        "  tokenizer.fit_on_texts(lines)\n",
        "  return(tokenizer.word_index)\n",
        "word2index=word_to_index(text_dict)\n",
        "idx2word = dict([(value, key) for key, value \\\n",
        "                 in word2index.items()]) \n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "s_w2WYEYJuVm",
        "colab_type": "text"
      },
      "source": [
        "### Max length"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "x75bUUfFhbth",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Max length\n",
        "def max_length(descriptions):\n",
        "  lines=[]\n",
        "  max_len=-1\n",
        "  for key in descriptions.keys():\n",
        "    for d in text_df[key]:\n",
        "      if len(d.split())>max_len:\n",
        "        max_len=len(d.split())\n",
        "  return max_len\n",
        "\n",
        "max_len=max_length(text_dict)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VhHRiZVkjop7",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "max_len"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4MvxZwfaJ8Ka",
        "colab_type": "text"
      },
      "source": [
        "### Implementing fastText embedding on text data\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fpu5AWX5ymx5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def load_vectors(fname):\n",
        "    fin = io.open(fname, 'r', encoding='utf-8',\\\n",
        "                  newline='\\n', errors='ignore')\n",
        "    embedding = dict()\n",
        "    for line in fin:\n",
        "        tokens = line.rstrip().split(' ')\n",
        "        embedding[tokens[0]] = np.array(tokens[1:],\\\n",
        "                                        dtype='float32')\n",
        "    return embedding\n",
        "embedding=load_vectors('/content/wiki.simple.vec')\n",
        "\n",
        "def fastText(embedding):\n",
        "  embedding_dim = 300\n",
        "  embedding_matrix = np.zeros((vocab_size+1, \\\n",
        "                               embedding_dim))\n",
        "  for word, i in word2index.items():\n",
        "      embedding_vector = embedding.get(word)\n",
        "      if embedding_vector is not None:\n",
        "          embedding_matrix[i] = embedding_vector\n",
        "  return embedding_matrix,embedding_dim\n",
        "\n",
        "embedding_matrix,embedding_dim=fastText(embedding)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "A4JDH5U1IPF7",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "text_values=np.array(tuple(text_dict.values()))\n",
        "text_idx=np.array(tuple(text_dict.keys()))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "V9rHnT9R3wRe",
        "colab_type": "text"
      },
      "source": [
        "### Creating Training Data\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vu-F0gJznC4E",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def create_train_data(text_values,text_idx,max_length,vocab_size):\n",
        "  X1,X2,y=list(),list(),list()\n",
        "  for i in range(0,len(text_idx)):\n",
        "    for line in text_values[i]:\n",
        "      numeric_seq = [word2index[word] for word in line.split()\\\n",
        "                     if word in word2index]\n",
        "    for ii in range(1,len(numeric_seq)):\n",
        "      in_seq,out_seq=numeric_seq[:ii],numeric_seq[ii]\n",
        "      in_seq=pad_sequences([in_seq],maxlen=max_length,\\\n",
        "                           padding='post')[0]\n",
        "      out_seq=to_categorical([out_seq],num_classes=vocab_size+1)[0]\n",
        "      X2.append(in_seq)\n",
        "      y.append(out_seq)\n",
        "      X1.append(text_idx[i])\n",
        "\n",
        "  return (X1),(X2),(y)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "E30RG7BnhmGc",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "X1,X2,y=create_train_data(text_values,text_idx,max_len,vocab_size)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qU0ou-5snD1x",
        "colab_type": "text"
      },
      "source": [
        "## Data Generators"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Kai5Uk_P-dha",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "batch_size=8\n",
        "step_per_epoch=len(X1)//batch_size\n",
        "epoch=30\n",
        "\n",
        "def data_generator(X1,X2,y,batch_size,epoch,\\\n",
        "                   step_size,max_len,vocab_size):\n",
        "  for j in range(0,epoch):\n",
        "    for k in range(0,step_size):      \n",
        "      for offset in range(0, len(X1), batch_size):\n",
        "        batch_X1,batchX2,batchY=list(),list(),list()\n",
        "        start_index=offset\n",
        "        end_index=offset+batch_size\n",
        "        for i in X1[start_index:end_index]:\n",
        "          batch_X1.append(features_df[i][0][0][0])\n",
        "        batchX2,batchY=X2[start_index:end_index],\\\n",
        "        y[start_index:end_index]\n",
        "        yield [np.array(batch_X1),np.array(batchX2)],\\\n",
        "        np.array(batchY)\n",
        "train_gen=data_generator(X1,X2,y,batch_size,epoch,\\\n",
        "                         step_per_epoch,max_len,vocab_size)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Zcbj7GpaKEC7",
        "colab_type": "text"
      },
      "source": [
        "## Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aT6zZkGInB1k",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Hyper params\n",
        "\n",
        "#image \n",
        "input_img = Input(shape=(2048,))\n",
        "feature_img1 = Dropout(0.5)(input_img)\n",
        "feature_img2 = Dense(256, activation='relu')(feature_img1)\n",
        "\n",
        "#text\n",
        "inputs_text = Input(shape=(max_len,))\n",
        "feature_text1 = Embedding(vocab_size+1,\\\n",
        "                          embedding_dim,name='embedding_layer')\\\n",
        "                          (inputs_text)\n",
        "feature_text2 = Dropout(0.5)(feature_text1)\n",
        "feature_text3 = GRU(256)(feature_text2)\n",
        "\n",
        "#Concat\n",
        "decoder1 = add([feature_img2, feature_text3])\n",
        "batch_norm=BatchNormalization(momentum=0.99, epsilon=0.001)(decoder1)\n",
        "decoder2 = Dense(256, activation='relu')(batch_norm)\n",
        "outputs = Dense(vocab_size+1, activation='softmax')(decoder2)\n",
        "\n",
        "model = Model(inputs=[input_img, inputs_text], outputs=outputs)\n",
        "model.get_layer('embedding_layer').set_weights([embedding_matrix])\n",
        "model.get_layer('embedding_layer').trainable = False\n",
        "\n",
        "model.compile(optimizer='adam',loss='categorical_crossentropy',\\\n",
        "              metrics=['accuracy'])\n",
        "model.summary()\n",
        "filepath = 'ImageCaptioningModel.h5'\n",
        "checkpoint = ModelCheckpoint(filepath, monitor='loss', \\\n",
        "                             save_best_only=True,mode='min')\n",
        "\n",
        "model.fit_generator(train_gen, epochs=epoch, steps_per_epoch=step_per_epoch, \\\n",
        "                    callbacks=[checkpoint],verbose=1)\n",
        "model.save_weights('/content/ImageCaptioningWeights.hdf5')\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oqrknHOnRIiH",
        "colab_type": "text"
      },
      "source": [
        "## Greedy Search"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pnrYRlgQKtug",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def greedy_search(model,photo,vocab,max_len):\n",
        "  output_text='start'\n",
        "  while True:\n",
        "    seq=[word2index[str(i)] for i in output_text.split()]\n",
        "    seq=pad_sequences([seq],maxlen=max_len,padding='post')\n",
        "    seq=np.array(seq[0])\n",
        "    y_predict=model.predict([np.expand_dims(photo,axis=0),\\\n",
        "                             np.expand_dims(seq,axis=0)])\n",
        "    output_text=output_text+' '+ idx2word[np.argmax(y_predict[0])]\n",
        "    if (len(output_text.split())>max_len) or \\\n",
        "    ((idx2word[np.argmax(y_predict[0])]) == 'end'):\n",
        "      break\n",
        "  output_text=output_text.split()\n",
        "  final_text=[' '.join(output_text[1:-1])]\n",
        "  return final_text    \n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DezttYIqNN1k",
        "colab_type": "text"
      },
      "source": [
        "## Result"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AWpEpuFuI6jM",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def test_feature_extractor(model,img_path):\n",
        "  test_features=list()\n",
        "  preproceesed_img=pre_process(img_path)\n",
        "  plt.imshow(np.squeeze(preproceesed_img))\n",
        "  plt.show()\n",
        "  model=InceptionV3(include_top=False,weights='imagenet')\n",
        "  for layer in model.layers:\n",
        "    layer.trainable=False\n",
        "  feature=model.predict(preproceesed_img)\n",
        "  test_features.append(feature)\n",
        "  return np.array(test_features[0][0][0][0])\n",
        "\n",
        "img_path='/content/image_caption.jpg'\n",
        "model=load_model('/content/ImageCaptioningModel.h5')\n",
        "\n",
        "test_feature=test_feature_extractor(model,img_path)\n",
        "\n",
        "sent=greedy_search(model,test_feature,vocab_size,max_len)\n",
        "print(\"Caption for image\",sent)"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}