8. WordPrediction.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-01T08:48:36.596604Z",
     "start_time": "2020-06-01T08:48:36.588602Z"
    }
   },
   "outputs": [],
   "source": [
    "# LSTM with Variable Length Input Sequences to One Character Output\n",
    "import numpy\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "from keras.layers import LSTM\n",
    "from keras.utils import np_utils\n",
    "from keras.preprocessing.sequence import pad_sequences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-01T08:48:37.541188Z",
     "start_time": "2020-06-01T08:48:37.536193Z"
    }
   },
   "outputs": [],
   "source": [
    "# fix random seed for reproducibility\n",
    "numpy.random.seed(7)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-01T08:48:38.062270Z",
     "start_time": "2020-06-01T08:48:38.058293Z"
    }
   },
   "outputs": [],
   "source": [
    "# define the raw dataset\n",
    "alphabet = \"ABCDEFGHIJKLMNOPQRSTUVWXYZ\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-01T08:48:38.393186Z",
     "start_time": "2020-06-01T08:48:38.387243Z"
    }
   },
   "outputs": [],
   "source": [
    "# create mapping of characters to integers (0-25) and the reverse\n",
    "char_to_int = dict((c, i) for i, c in enumerate(alphabet))\n",
    "int_to_char = dict((i, c) for i, c in enumerate(alphabet))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-01T08:48:39.459719Z",
     "start_time": "2020-06-01T08:48:39.388718Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('QR', '->', 'S', 'BCDEF', '->', 'G', 'PQRS', '->', 'T', 'BC', '->', 'D', 'TUVW', '->', 'X', 'DEFGH', '->', 'I', 'TU', '->', 'V', 'B', '->', 'C', 'VW', '->', 'X', 'MN', '->', 'O', 'CDEFG', '->', 'H', 'DEFG', '->', 'H', 'QRST', '->', 'U', 'RS', '->', 'T', 'OPQ', '->', 'R', 'KLMNO', '->', 'P', 'GHIJ', '->', 'K', 'STUV', '->', 'W', 'OPQRS', '->', 'T', 'L', '->', 'M', 'GHIJ', '->', 'K', 'JKL', '->', 'M', 'AB', '->', 'C', 'OP', '->', 'Q', 'IJ', '->', 'K', 'ABC', '->', 'D', 'HIJK', '->', 'L', 'IJKLM', '->', 'N', 'TUVWX', '->', 'Y', 'R', '->', 'S', 'X', '->', 'Y', 'IJKLM', '->', 'N', 'S', '->', 'T', 'PQ', '->', 'R', 'DEF', '->', 'G', 'KLMNO', '->', 'P', 'NOP', '->', 'Q', 'EF', '->', 'G', 'M', '->', 'N', 'KLMN', '->', 'O', 'NO', '->', 'P', 'ST', '->', 'U', 'C', '->', 'D', 'QR', '->', 'S', 'D', '->', 'E', 'KLM', '->', 'N', 'LMN', '->', 'O', 'X', '->', 'Y', 'STU', '->', 'V', 'WX', '->', 'Y')\n"
     ]
    }
   ],
   "source": [
    "# prepare the dataset of input to output pairs encoded as integers\n",
    "num_inputs = 1000\n",
    "max_len = 5\n",
    "dataX = []\n",
    "dataY = []\n",
    "for i in range(num_inputs):\n",
    "    start = numpy.random.randint(len(alphabet)-2)\n",
    "    end = numpy.random.randint(start, min(start+max_len,len(alphabet)-1))\n",
    "    sequence_in = alphabet[start:end+1]\n",
    "    sequence_out = alphabet[end + 1]\n",
    "    dataX.append([char_to_int[char] for char in sequence_in])\n",
    "    dataY.append(char_to_int[sequence_out])\n",
    "    seq += sequence_in, '->', sequence_out\n",
    "print(seq[:150])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-01T08:48:41.054241Z",
     "start_time": "2020-06-01T08:48:41.025213Z"
    }
   },
   "outputs": [],
   "source": [
    "# convert list of lists to array and pad sequences if needed\n",
    "X = pad_sequences(dataX, maxlen=max_len, dtype='float32')\n",
    "# reshape X to be [samples, time steps, features]\n",
    "X = numpy.reshape(X, (X.shape[0], max_len, 1))\n",
    "# normalize\n",
    "X = X / float(len(alphabet))\n",
    "# one hot encode the output variable\n",
    "y = np_utils.to_categorical(dataY)\n",
    "# create and fit the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-01T09:07:16.913482Z",
     "start_time": "2020-06-01T09:06:14.382881Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.callbacks.History at 0x1f4c3398e48>"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch_size = 50\n",
    "model = Sequential()\n",
    "model.add(LSTM(32, input_shape=(X.shape[1], 1)))\n",
    "model.add(Dense(y.shape[1], activation='softmax'))\n",
    "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "model.fit(X, y, epochs=500, batch_size=batch_size, verbose=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-01T09:07:20.184866Z",
     "start_time": "2020-06-01T09:07:19.514787Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model Accuracy: 88.50%\n"
     ]
    }
   ],
   "source": [
    "# summarize performance of the model\n",
    "scores = model.evaluate(X, y, verbose=0)\n",
    "print(\"Model Accuracy: %.2f%%\" % (scores[1]*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-01T09:07:25.778761Z",
     "start_time": "2020-06-01T09:07:25.121659Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['I'] -> J\n",
      "['G', 'H', 'I', 'J', 'K'] -> L\n",
      "['U', 'V', 'W', 'X'] -> Y\n",
      "['T', 'U', 'V', 'W', 'X'] -> Y\n",
      "['T', 'U', 'V'] -> W\n",
      "['V', 'W', 'X', 'Y'] -> Z\n",
      "['J', 'K'] -> L\n",
      "['H', 'I', 'J', 'K'] -> L\n",
      "['W', 'X'] -> Y\n",
      "['K', 'L', 'M', 'N', 'O'] -> P\n",
      "['C', 'D'] -> E\n",
      "['M'] -> N\n",
      "['V', 'W', 'X'] -> Y\n",
      "['N'] -> N\n",
      "['T', 'U', 'V', 'W'] -> X\n",
      "['Q'] -> R\n",
      "['I', 'J', 'K'] -> L\n",
      "['I'] -> J\n",
      "['J', 'K', 'L', 'M', 'N'] -> O\n",
      "['M'] -> N\n"
     ]
    }
   ],
   "source": [
    "# demonstrate some model predictions\n",
    "for i in range(20):\n",
    "    pattern_index = numpy.random.randint(len(dataX))\n",
    "    pattern = dataX[pattern_index]\n",
    "    x = pad_sequences([pattern], maxlen=max_len, dtype='float32')\n",
    "    x = numpy.reshape(x, (1, max_len, 1))\n",
    "    x = x / float(len(alphabet))\n",
    "    prediction = model.predict(x, verbose=0)\n",
    "    index = numpy.argmax(prediction)\n",
    "    result = int_to_char[index]\n",
    "    seq_in = [int_to_char[value] for value in pattern]\n",
    "    print(seq_in, \"->\", result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}