11-glove-and-phonetics-applied.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Combining glove and phonetic embedding to get a better accuracy in predicting new rap lyrics\n",
    "## The idea is to combine our previous results with the phonetics of a word <br/> and get more detailed information about the structure of a sentence which rhymes. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import zipfile\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from tensorboard.plugins import projector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tools.processing as pre\n",
    "import tools.embedding as emb\n",
    "import tools.architectures as nn\n",
    "import tools.training as tr\n",
    "import tools.phonetics as phon"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create embedding for rap lyrics"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We are only using halve of the words \"250000\" because of memory issues"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 3.20919529e-02  6.76626116e-02  4.15114537e-02 ... -4.64479961e-02\n",
      "  -3.84210385e-02 -4.82199974e-02]\n",
      " [-2.10926658e-03  1.25741512e-02 -1.79006178e-02 ... -2.81850225e-05\n",
      "   5.56626776e-03  3.17836367e-03]\n",
      " [ 8.25334620e-03  4.66938363e-03 -3.94539274e-02 ... -5.67189045e-02\n",
      "  -2.74822619e-02  5.31177968e-02]\n",
      " ...\n",
      " [-3.14446986e-02 -1.60996541e-02 -1.04854435e-01 ... -3.80562693e-02\n",
      "   1.62324663e-02  1.56729072e-02]\n",
      " [-1.65838236e-03 -6.00066036e-04 -8.61928016e-02 ...  3.56212556e-02\n",
      "  -1.71570492e-03 -4.26971316e-02]\n",
      " [ 3.77826095e-02 -1.21965548e-02 -5.14048897e-02 ...  5.14076790e-03\n",
      "  -3.03092636e-02 -4.75349426e-02]]\n"
     ]
    }
   ],
   "source": [
    "text = pre.get_text(\"data/cleaned-rap-lyrics/final_2_pac_rakim_kid_cudi.txt\")[:25000]\n",
    "\n",
    "vocab = pre.Vocabulary(text)\n",
    "word2index = vocab.word2index\n",
    "index2word = vocab.index2word\n",
    "VOCAB_SIZE = len(index2word)\n",
    "\n",
    "# create embedding for words\n",
    "word_embedding_matrix = emb.get_embedding_matrix(word2index, VOCAB_SIZE)\n",
    "print(word_embedding_matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "35\n"
     ]
    }
   ],
   "source": [
    "print(VOCAB_SIZE)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create embedding for phonems"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[-0.83106965 -0.53812933  0.14049865]\n",
      " [-0.57452571 -0.45615974  0.67958683]\n",
      " [-0.85238743 -0.27057332 -0.4474659 ]\n",
      " [-0.89024484 -0.43297064  0.14142321]\n",
      " [-0.889552   -0.42086163  0.17768735]\n",
      " [-0.84767091  0.20958452 -0.48736873]\n",
      " [-0.80071777 -0.59023607 -0.10233522]\n",
      " [-0.8041234  -0.43067229  0.40976447]\n",
      " [-0.85063279 -0.25427106 -0.46018487]\n",
      " [-0.83106846 -0.49368501 -0.25612572]\n",
      " [-0.77628517 -0.57696062 -0.25396407]\n",
      " [-0.85525501  0.34369221  0.38783291]\n",
      " [-0.72286534 -0.66972053  0.17011791]\n",
      " [-0.90681714  0.07853168 -0.41414431]\n",
      " [-0.84486431 -0.38346329 -0.37304166]\n",
      " [-0.73923445 -0.57153291 -0.35620561]\n",
      " [-0.94168699  0.08752931  0.32490653]\n",
      " [ 0.91624618 -0.1400506  -0.37533814]\n",
      " [-0.88760549 -0.45781818 -0.05058585]\n",
      " [-0.95192575  0.20271704 -0.22965866]\n",
      " [-0.88998002  0.08347418 -0.44829413]\n",
      " [-0.1288805  -0.28575739 -0.94959605]\n",
      " [-0.87844598 -0.46678743 -0.10218589]\n",
      " [-0.99973291  0.0226963   0.00435353]\n",
      " [-0.92469871  0.07278427  0.37367749]\n",
      " [-0.92872733  0.10954829  0.35420984]\n",
      " [-0.98819238  0.08642044  0.12651959]\n",
      " [-0.88798094 -0.05522604  0.45655221]\n",
      " [-0.71483773 -0.66277325  0.22302185]\n",
      " [-0.77870333  0.09508178  0.62014556]\n",
      " [-0.82572544 -0.36834463 -0.42719984]\n",
      " [-0.96458721 -0.03701936  0.26115352]\n",
      " [-0.75869954 -0.52236611  0.3892411 ]\n",
      " [-0.89848369 -0.42166027 -0.12218721]\n",
      " [-0.85189265 -0.49740666 -0.16390714]\n",
      " [-0.08865134  0.95458961  0.28442836]\n",
      " [-0.92101055 -0.30935183 -0.23673016]\n",
      " [-0.9315502  -0.20965256  0.29708615]\n",
      " [-0.82505924 -0.51443595 -0.23373653]\n",
      " [-0.83033556 -0.46001527  0.3145296 ]\n",
      " [-0.94038957  0.13505612  0.31213349]\n",
      " [-0.91232622  0.25985038  0.31644672]\n",
      " [-0.89272779  0.00176275  0.45059285]\n",
      " [-0.87289619 -0.1406185  -0.46720299]\n",
      " [-0.65745056 -0.73210901  0.17825575]\n",
      " [-0.80017054 -0.57691598 -0.16399632]\n",
      " [-0.92555779 -0.04088644 -0.37639201]\n",
      " [-0.91765714  0.09767944 -0.38518035]\n",
      " [-0.955446   -0.16260955 -0.2463353 ]\n",
      " [-0.84971929 -0.46367732 -0.25095904]\n",
      " [-0.83528256  0.25620309  0.48648021]\n",
      " [-0.90798557  0.01761704  0.41863087]\n",
      " [-0.92834795  0.13178164  0.34756806]\n",
      " [-0.92108727  0.05003754  0.38612777]\n",
      " [-0.85894579 -0.10602354  0.5009703 ]\n",
      " [-0.94464797  0.2781046   0.17406319]\n",
      " [-0.90446991  0.4144173  -0.10095845]\n",
      " [-0.83686346 -0.54681742 -0.02549712]\n",
      " [-0.96190029  0.15010665 -0.22850786]\n",
      " [-0.89484626 -0.25758079 -0.36455774]\n",
      " [-0.88388461 -0.35842159 -0.30046943]\n",
      " [-0.79716337 -0.2344465   0.55638605]\n",
      " [ 0.81084776 -0.57908005  0.08480806]\n",
      " [-0.9342941   0.33684736 -0.116741  ]\n",
      " [-0.77812654 -0.12117996  0.61630702]\n",
      " [-0.9283123   0.06958895  0.36523083]\n",
      " [-0.95217842  0.22846287  0.20288117]\n",
      " [-0.91551638 -0.12399001  0.38269579]\n",
      " [-0.89139616  0.03926151  0.45152137]\n",
      " [-0.96463948  0.15677732  0.21187623]\n",
      " [-0.93844485  0.07164304  0.3379181 ]\n",
      " [ 0.10141908 -0.98955452 -0.10245005]\n",
      " [ 0.7182911  -0.63876063 -0.2757585 ]\n",
      " [-0.91440481 -0.31957087 -0.24847195]\n",
      " [-0.81720048 -0.47572622  0.32537353]\n",
      " [-0.47011715 -0.12233038 -0.87408537]\n",
      " [-0.88914901 -0.15891369 -0.42913917]\n",
      " [-0.86603016  0.03746723 -0.49858603]\n",
      " [-0.68690002  0.23986557  0.68602699]\n",
      " [-0.89934129  0.12577032  0.4187685 ]\n",
      " [-0.8662762  -0.01368999  0.49937788]\n",
      " [-0.92921001  0.3657164  -0.05310719]\n",
      " [-0.95402688  0.04104564  0.29689705]\n",
      " [-0.53371918 -0.81742424 -0.2167059 ]\n",
      " [-0.77139229 -0.16222043  0.03486567]]\n"
     ]
    }
   ],
   "source": [
    "phonems = pre.get_text(\"data/phonem-rap-lyrics/phonem_all.txt\")\n",
    "\n",
    "vocab_phonem = pre.Vocabulary(phonems)\n",
    "phonem2index = vocab_phonem.word2index\n",
    "index2phonem = vocab_phonem.index2word\n",
    "VOCAB_SIZE_PHONEM = len(index2phonem)\n",
    "\n",
    "# create embedding for phonems\n",
    "phonem_embedding_matrix = emb.get_phonem_embedding_matrix(phonem2index, VOCAB_SIZE_PHONEM)\n",
    "print(phonem_embedding_matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "85\n"
     ]
    }
   ],
   "source": [
    "print(VOCAB_SIZE_PHONEM)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Preparation: Split sentences of rap lyrics into data and label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(['yeah', 'you', 'know', 'what', 'this', 'is', 'nyc', ';', 'the', 'triumphant', 'return', 'rakim', 'allah', ';', 'rakim', ';'], 'remember'), (['you', 'know', 'what', 'this', 'is', 'nyc', ';', 'the', 'triumphant', 'return', 'rakim', 'allah', ';', 'rakim', ';', 'remember'], 'being'), (['know', 'what', 'this', 'is', 'nyc', ';', 'the', 'triumphant', 'return', 'rakim', 'allah', ';', 'rakim', ';', 'remember', 'being'], 'introduced'), (['what', 'this', 'is', 'nyc', ';', 'the', 'triumphant', 'return', 'rakim', 'allah', ';', 'rakim', ';', 'remember', 'being', 'introduced'], 'to'), (['this', 'is', 'nyc', ';', 'the', 'triumphant', 'return', 'rakim', 'allah', ';', 'rakim', ';', 'remember', 'being', 'introduced', 'to'], 'rapping')]\n"
     ]
    }
   ],
   "source": [
    "word_tokens = text.split()\n",
    "\n",
    "TIMESTEPS = 16\n",
    "\n",
    "str_data, str_labels = pre.create_data_label_pairs(word_tokens, TIMESTEPS)\n",
    "\n",
    "print( list( zip(str_data, str_labels) )[:5] )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Preparation: Convert previous data into phonems and keep the label"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Update arpabet dictionary of phonems with unknown words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n"
     ]
    }
   ],
   "source": [
    "unknown_list = pre.get_text(\"data/cleaned-rap-lyrics/unknown_words_dict.txt\")\n",
    "unknown_dict = phon.create_unknown_dict_from_text(unknown_list)\n",
    "\n",
    "phon.update_arpabet(unknown_dict)\n",
    "print(len(phon.get_unknown_words()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(['Y', 'AE1', 'Y', 'UW1', 'N', 'OW1', 'W', 'AH1', 'T', 'DH', 'IH1', 'S', 'IH1', 'Z', 'N', 'IH', 'K', 'DH', 'AH0', 'T', 'R', 'AY0', 'AH1', 'M', 'F', 'AH0', 'N', 'T', 'R', 'IH0', 'T', 'ER1', 'N', 'R', 'AE', 'K', 'AH', 'M', 'AA1', 'L', 'AH0', 'R', 'AE', 'K', 'AH', 'M'], 'remember')]\n"
     ]
    }
   ],
   "source": [
    "phonem_data = []\n",
    "for sent in str_data:\n",
    "    phon_str = []\n",
    "    for word in sent:\n",
    "        if not word == \";\":\n",
    "            phon_str.extend(phon.get_phonem(word))\n",
    "    phonem_data.append(phon_str)\n",
    "    \n",
    "print( list( zip(phonem_data, str_labels) )[:1] )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### The rap lyrics and phonems are encoded so that their shape fits into the embedding lookup table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# words\n",
    "encoder = tr.IndexWordEncoder(\"Index-Word-Encoding\", word2index)\n",
    "decoder = tr.OneHotWordDecoder(\"1-Hot-Word-Decoding\", index2word, temperature=0.8)\n",
    "# phonem\n",
    "encoder_phonem = tr.IndexWordEncoder(\"Index-Phonem-Encoding\", phonem2index)\n",
    "\n",
    "data = encoder.encode(str_data)\n",
    "data_phonem = encoder_phonem.encode(phonem_data)\n",
    "labels = encoder.encode_labels(str_labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Delete unneeded variables to save space"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "del str_labels\n",
    "del str_data\n",
    "del phonem_data\n",
    "del word_tokens"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### In order to get the correct shape of the phonem embedding matrix, we needed to pad missing entries <br/> The changes were made in the library file: training.py:IndexWordEncoder()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(31, 16)\n",
      "(31, 35)\n",
      "(31, 66)\n"
     ]
    }
   ],
   "source": [
    "print(data.shape)\n",
    "print(labels.shape)\n",
    "print(data_phonem.shape)\n",
    "\n",
    "# choose timesteps according to length of padded phonem array\n",
    "TIMESTEPS_PHONEM = len(data_phonem[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tensorflow Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "PRE_TRAINED = True\n",
    "EPOCHS = 10\n",
    "BATCH_SIZE = 128\n",
    "\n",
    "embedding_dimension = emb.GLOVE_SIZE\n",
    "embedding_dimension_phonem = emb.PHONEM_SIZE\n",
    "\n",
    "hidden_layer_size = 32\n",
    "hidden_layer_size_phonem = 16\n",
    "\n",
    "num_LSTM_layers = 4\n",
    "num_LSTM_layers_phonem = 4"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Build a model with both embeddings combined"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "seed_text = \"yeah you know what this is nyc ; the triumphant return rakim allah ; rakim ; remember being introduced to rapping your first rhyme ;\"\n",
    "sampler = lambda trainable, _: tr.sample(seed_text, trainable, encoder, decoder, length=20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### [Error] We are getting an error during the build model process. <br/> We were unable to find a solution to that..\n",
    "#### >> Trying to share variable lstm-words-phonems/rnn/multi_rnn_cell/cell_0/lstm_cell/kernel, but specified shape (19, 64) and found shape (332, 128) <<"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /Users/rene/workspace/Tensorflow/.pyenv/lib/python3.7/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "WARNING:tensorflow:From /Users/rene/workspace/Tensorflow/Seminar NLP/Project/deep-rap/tools/architectures.py:242: LSTMCell.__init__ (from tensorflow.python.ops.rnn_cell_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This class is equivalent as tf.keras.layers.LSTMCell, and will be replaced by that in Tensorflow 2.0.\n",
      "\n",
      "WARNING: The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
      "For more information, please see:\n",
      "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
      "  * https://github.com/tensorflow/addons\n",
      "If you depend on functionality not listed there, please file an issue.\n",
      "\n",
      "WARNING:tensorflow:From /Users/rene/workspace/Tensorflow/Seminar NLP/Project/deep-rap/tools/architectures.py:245: MultiRNNCell.__init__ (from tensorflow.python.ops.rnn_cell_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This class is equivalent as tf.keras.layers.StackedRNNCells, and will be replaced by that in Tensorflow 2.0.\n",
      "WARNING:tensorflow:From /Users/rene/workspace/Tensorflow/Seminar NLP/Project/deep-rap/tools/architectures.py:562: dynamic_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `keras.layers.RNN(cell)`, which is equivalent to this API\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "Trying to share variable lstm-words-phonems/rnn/multi_rnn_cell/cell_0/lstm_cell/kernel, but specified shape (19, 64) and found shape (332, 128).",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-15-88dee633e385>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      2\u001b[0m rnn.build(num_LSTM_layers, hidden_layer_size, VOCAB_SIZE, TIMESTEPS, embedding_dimension,\n\u001b[1;32m      3\u001b[0m           \u001b[0mnum_LSTM_layers_phonem\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhidden_layer_size_phonem\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mVOCAB_SIZE_PHONEM\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mTIMESTEPS_PHONEM\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0membedding_dimension_phonem\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m           l2_reg=0.0)\n\u001b[0m\u001b[1;32m      5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m tr.train_model_more_embeddings(rnn, data, data_phonem, labels,\n",
      "\u001b[0;32m~/workspace/Tensorflow/Seminar NLP/Project/deep-rap/tools/architectures.py\u001b[0m in \u001b[0;36mbuild\u001b[0;34m(self, num_layers, hidden_layer_size, vocab_size, time_steps, embedding_words, num_layers_phonem, hidden_layer_size_phonem, vocab_size_phonem, time_steps_phonem, embedding_phonems, l2_reg)\u001b[0m\n\u001b[1;32m    567\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstacked_cells_phonem\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlstm_layer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnum_layers_phonem\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhidden_layer_size_phonem\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    568\u001b[0m                 self.outputs_phonem, self.states_phonem = tf.nn.dynamic_rnn(\n\u001b[0;32m--> 569\u001b[0;31m                         self.stacked_cells_phonem, embed_phonem, sequence_length=None, dtype=tf.float32)\n\u001b[0m\u001b[1;32m    570\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlast_rnn_output_phonem\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstates_phonem\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mnum_layers_phonem\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    571\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/workspace/Tensorflow/.pyenv/lib/python3.7/site-packages/tensorflow/python/util/deprecation.py\u001b[0m in \u001b[0;36mnew_func\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    322\u001b[0m               \u001b[0;34m'in a future version'\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mdate\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;34m'after %s'\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mdate\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    323\u001b[0m               instructions)\n\u001b[0;32m--> 324\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    325\u001b[0m     return tf_decorator.make_decorator(\n\u001b[1;32m    326\u001b[0m         \u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnew_func\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'deprecated'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/workspace/Tensorflow/.pyenv/lib/python3.7/site-packages/tensorflow/python/ops/rnn.py\u001b[0m in \u001b[0;36mdynamic_rnn\u001b[0;34m(cell, inputs, sequence_length, initial_state, dtype, parallel_iterations, swap_memory, time_major, scope)\u001b[0m\n\u001b[1;32m    669\u001b[0m         \u001b[0mswap_memory\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mswap_memory\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    670\u001b[0m         \u001b[0msequence_length\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msequence_length\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 671\u001b[0;31m         dtype=dtype)\n\u001b[0m\u001b[1;32m    672\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    673\u001b[0m     \u001b[0;31m# Outputs of _dynamic_rnn_loop are always shaped [time, batch, depth].\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/workspace/Tensorflow/.pyenv/lib/python3.7/site-packages/tensorflow/python/ops/rnn.py\u001b[0m in \u001b[0;36m_dynamic_rnn_loop\u001b[0;34m(cell, inputs, initial_state, parallel_iterations, swap_memory, sequence_length, dtype)\u001b[0m\n\u001b[1;32m    877\u001b[0m       \u001b[0mparallel_iterations\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mparallel_iterations\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    878\u001b[0m       \u001b[0mmaximum_iterations\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtime_steps\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 879\u001b[0;31m       swap_memory=swap_memory)\n\u001b[0m\u001b[1;32m    880\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    881\u001b[0m   \u001b[0;31m# Unpack final output if not using output tuples.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/workspace/Tensorflow/.pyenv/lib/python3.7/site-packages/tensorflow/python/ops/control_flow_ops.py\u001b[0m in \u001b[0;36mwhile_loop\u001b[0;34m(cond, body, loop_vars, shape_invariants, parallel_iterations, back_prop, swap_memory, name, maximum_iterations, return_same_structure)\u001b[0m\n\u001b[1;32m   3554\u001b[0m       \u001b[0mops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd_to_collection\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mGraphKeys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mWHILE_CONTEXT\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mloop_context\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3555\u001b[0m     result = loop_context.BuildLoop(cond, body, loop_vars, shape_invariants,\n\u001b[0;32m-> 3556\u001b[0;31m                                     return_same_structure)\n\u001b[0m\u001b[1;32m   3557\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mmaximum_iterations\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3558\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/workspace/Tensorflow/.pyenv/lib/python3.7/site-packages/tensorflow/python/ops/control_flow_ops.py\u001b[0m in \u001b[0;36mBuildLoop\u001b[0;34m(self, pred, body, loop_vars, shape_invariants, return_same_structure)\u001b[0m\n\u001b[1;32m   3085\u001b[0m       \u001b[0;32mwith\u001b[0m \u001b[0mops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_default_graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_mutation_lock\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# pylint: disable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3086\u001b[0m         original_body_result, exit_vars = self._BuildLoop(\n\u001b[0;32m-> 3087\u001b[0;31m             pred, body, original_loop_vars, loop_vars, shape_invariants)\n\u001b[0m\u001b[1;32m   3088\u001b[0m     \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3089\u001b[0m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mExit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/workspace/Tensorflow/.pyenv/lib/python3.7/site-packages/tensorflow/python/ops/control_flow_ops.py\u001b[0m in \u001b[0;36m_BuildLoop\u001b[0;34m(self, pred, body, original_loop_vars, loop_vars, shape_invariants)\u001b[0m\n\u001b[1;32m   3020\u001b[0m         flat_sequence=vars_for_body_with_tensor_arrays)\n\u001b[1;32m   3021\u001b[0m     \u001b[0mpre_summaries\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_collection\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mGraphKeys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_SUMMARY_COLLECTION\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# pylint: disable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3022\u001b[0;31m     \u001b[0mbody_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbody\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mpacked_vars_for_body\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   3023\u001b[0m     \u001b[0mpost_summaries\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_collection\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mGraphKeys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_SUMMARY_COLLECTION\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# pylint: disable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3024\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mnest\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_sequence\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbody_result\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/workspace/Tensorflow/.pyenv/lib/python3.7/site-packages/tensorflow/python/ops/control_flow_ops.py\u001b[0m in \u001b[0;36m<lambda>\u001b[0;34m(i, lv)\u001b[0m\n\u001b[1;32m   3523\u001b[0m         cond = lambda i, lv: (  # pylint: disable=g-long-lambda\n\u001b[1;32m   3524\u001b[0m             math_ops.logical_and(i < maximum_iterations, orig_cond(*lv)))\n\u001b[0;32m-> 3525\u001b[0;31m         \u001b[0mbody\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mlambda\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlv\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0morig_body\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mlv\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   3526\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3527\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mcontext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexecuting_eagerly\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/workspace/Tensorflow/.pyenv/lib/python3.7/site-packages/tensorflow/python/ops/rnn.py\u001b[0m in \u001b[0;36m_time_step\u001b[0;34m(time, output_ta_t, state)\u001b[0m\n\u001b[1;32m    845\u001b[0m           skip_conditionals=True)\n\u001b[1;32m    846\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 847\u001b[0;31m       \u001b[0;34m(\u001b[0m\u001b[0moutput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnew_state\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcall_cell\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    848\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    849\u001b[0m     \u001b[0;31m# Keras cells always wrap state as list, even if it's a single tensor.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/workspace/Tensorflow/.pyenv/lib/python3.7/site-packages/tensorflow/python/ops/rnn.py\u001b[0m in \u001b[0;36m<lambda>\u001b[0;34m()\u001b[0m\n\u001b[1;32m    831\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mis_keras_rnn_cell\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mnest\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_sequence\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    832\u001b[0m       \u001b[0mstate\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 833\u001b[0;31m     \u001b[0mcall_cell\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mlambda\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mcell\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput_t\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstate\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    834\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    835\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0msequence_length\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/workspace/Tensorflow/.pyenv/lib/python3.7/site-packages/tensorflow/python/ops/rnn_cell_impl.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs, state, scope)\u001b[0m\n\u001b[1;32m    232\u001b[0m         \u001b[0msetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mscope_attrname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mscope\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    233\u001b[0m       \u001b[0;32mwith\u001b[0m \u001b[0mscope\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 234\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mRNNCell\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__call__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstate\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    235\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    236\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_rnn_get_variable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgetter\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/workspace/Tensorflow/.pyenv/lib/python3.7/site-packages/tensorflow/python/layers/base.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs, *args, **kwargs)\u001b[0m\n\u001b[1;32m    528\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    529\u001b[0m       \u001b[0;31m# Actually call layer\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 530\u001b[0;31m       \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mLayer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__call__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    531\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    532\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mcontext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexecuting_eagerly\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/workspace/Tensorflow/.pyenv/lib/python3.7/site-packages/tensorflow/python/keras/engine/base_layer.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs, *args, **kwargs)\u001b[0m\n\u001b[1;32m    552\u001b[0m             \u001b[0;31m# In graph mode, failure to build the layer's graph\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    553\u001b[0m             \u001b[0;31m# implies a user-side bug. We don't catch exceptions.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 554\u001b[0;31m             \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcall\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    555\u001b[0m           \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    556\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/workspace/Tensorflow/.pyenv/lib/python3.7/site-packages/tensorflow/python/ops/rnn_cell_impl.py\u001b[0m in \u001b[0;36mcall\u001b[0;34m(self, inputs, state)\u001b[0m\n\u001b[1;32m   1513\u001b[0m                                       [-1, cell.state_size])\n\u001b[1;32m   1514\u001b[0m           \u001b[0mcur_state_pos\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mcell\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstate_size\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1515\u001b[0;31m         \u001b[0mcur_inp\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnew_state\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcell\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcur_inp\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcur_state\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1516\u001b[0m         \u001b[0mnew_states\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnew_state\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1517\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/workspace/Tensorflow/.pyenv/lib/python3.7/site-packages/tensorflow/python/ops/rnn_cell_impl.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs, state, scope, *args, **kwargs)\u001b[0m\n\u001b[1;32m    369\u001b[0m     \u001b[0;31m# method.  See the class docstring for more details.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    370\u001b[0m     return base_layer.Layer.__call__(self, inputs, state, scope=scope,\n\u001b[0;32m--> 371\u001b[0;31m                                      *args, **kwargs)\n\u001b[0m\u001b[1;32m    372\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    373\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/workspace/Tensorflow/.pyenv/lib/python3.7/site-packages/tensorflow/python/layers/base.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs, *args, **kwargs)\u001b[0m\n\u001b[1;32m    528\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    529\u001b[0m       \u001b[0;31m# Actually call layer\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 530\u001b[0;31m       \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mLayer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__call__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    531\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    532\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mcontext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexecuting_eagerly\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/workspace/Tensorflow/.pyenv/lib/python3.7/site-packages/tensorflow/python/keras/engine/base_layer.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs, *args, **kwargs)\u001b[0m\n\u001b[1;32m    536\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbuilt\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    537\u001b[0m         \u001b[0;31m# Build layer if applicable (if the `build` method has been overridden).\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 538\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_maybe_build\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    539\u001b[0m         \u001b[0;31m# We must set self.built since user defined build functions are not\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    540\u001b[0m         \u001b[0;31m# constrained to set self.built.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/workspace/Tensorflow/.pyenv/lib/python3.7/site-packages/tensorflow/python/keras/engine/base_layer.py\u001b[0m in \u001b[0;36m_maybe_build\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   1601\u001b[0m     \u001b[0;31m# Only call `build` if the user has manually overridden the build method.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1602\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mhasattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbuild\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'_is_default'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1603\u001b[0;31m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbuild\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput_shapes\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1604\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1605\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m__setattr__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/workspace/Tensorflow/.pyenv/lib/python3.7/site-packages/tensorflow/python/keras/utils/tf_utils.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(instance, input_shape)\u001b[0m\n\u001b[1;32m    149\u001b[0m       \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    150\u001b[0m         \u001b[0minput_shape\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtuple\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtensor_shape\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTensorShape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput_shape\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_list\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 151\u001b[0;31m     \u001b[0moutput_shape\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minstance\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput_shape\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    152\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0moutput_shape\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    153\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput_shape\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/workspace/Tensorflow/.pyenv/lib/python3.7/site-packages/tensorflow/python/ops/rnn_cell_impl.py\u001b[0m in \u001b[0;36mbuild\u001b[0;34m(self, inputs_shape)\u001b[0m\n\u001b[1;32m    925\u001b[0m         \u001b[0mshape\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0minput_depth\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mh_depth\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m4\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_num_units\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    926\u001b[0m         \u001b[0minitializer\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_initializer\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 927\u001b[0;31m         partitioner=maybe_partitioner)\n\u001b[0m\u001b[1;32m    928\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdtype\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    929\u001b[0m       \u001b[0minitializer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0minit_ops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzeros_initializer\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/workspace/Tensorflow/.pyenv/lib/python3.7/site-packages/tensorflow/python/keras/engine/base_layer.py\u001b[0m in \u001b[0;36madd_variable\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1230\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0madd_variable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1231\u001b[0m     \u001b[0;34m\"\"\"Alias for `add_weight`.\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1232\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd_weight\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1233\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1234\u001b[0m   \u001b[0;34m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/workspace/Tensorflow/.pyenv/lib/python3.7/site-packages/tensorflow/python/layers/base.py\u001b[0m in \u001b[0;36madd_weight\u001b[0;34m(self, name, shape, dtype, initializer, regularizer, trainable, constraint, use_resource, synchronization, aggregation, partitioner)\u001b[0m\n\u001b[1;32m    433\u001b[0m             \u001b[0msynchronization\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msynchronization\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    434\u001b[0m             \u001b[0maggregation\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0maggregation\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 435\u001b[0;31m             getter=vs.get_variable)\n\u001b[0m\u001b[1;32m    436\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    437\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mregularizer\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/workspace/Tensorflow/.pyenv/lib/python3.7/site-packages/tensorflow/python/keras/engine/base_layer.py\u001b[0m in \u001b[0;36madd_weight\u001b[0;34m(self, name, shape, dtype, initializer, regularizer, trainable, constraint, partitioner, use_resource, synchronization, aggregation, **kwargs)\u001b[0m\n\u001b[1;32m    347\u001b[0m         \u001b[0mcollections\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcollections\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    348\u001b[0m         \u001b[0msynchronization\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msynchronization\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 349\u001b[0;31m         aggregation=aggregation)\n\u001b[0m\u001b[1;32m    350\u001b[0m     \u001b[0mbackend\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrack_variable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvariable\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    351\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/workspace/Tensorflow/.pyenv/lib/python3.7/site-packages/tensorflow/python/training/checkpointable/base.py\u001b[0m in \u001b[0;36m_add_variable_with_custom_getter\u001b[0;34m(self, name, shape, dtype, initializer, getter, overwrite, **kwargs_for_getter)\u001b[0m\n\u001b[1;32m    605\u001b[0m     new_variable = getter(\n\u001b[1;32m    606\u001b[0m         \u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mshape\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minitializer\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minitializer\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 607\u001b[0;31m         **kwargs_for_getter)\n\u001b[0m\u001b[1;32m    608\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    609\u001b[0m     \u001b[0;31m# If we set an initializer and the variable processed it, tracking will not\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/workspace/Tensorflow/.pyenv/lib/python3.7/site-packages/tensorflow/python/ops/variable_scope.py\u001b[0m in \u001b[0;36mget_variable\u001b[0;34m(name, shape, dtype, initializer, regularizer, trainable, collections, caching_device, partitioner, validate_shape, use_resource, custom_getter, constraint, synchronization, aggregation)\u001b[0m\n\u001b[1;32m   1477\u001b[0m       \u001b[0mconstraint\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mconstraint\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1478\u001b[0m       \u001b[0msynchronization\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msynchronization\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1479\u001b[0;31m       aggregation=aggregation)\n\u001b[0m\u001b[1;32m   1480\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1481\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/workspace/Tensorflow/.pyenv/lib/python3.7/site-packages/tensorflow/python/ops/variable_scope.py\u001b[0m in \u001b[0;36mget_variable\u001b[0;34m(self, var_store, name, shape, dtype, initializer, regularizer, reuse, trainable, collections, caching_device, partitioner, validate_shape, use_resource, custom_getter, constraint, synchronization, aggregation)\u001b[0m\n\u001b[1;32m   1218\u001b[0m           \u001b[0mconstraint\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mconstraint\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1219\u001b[0m           \u001b[0msynchronization\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msynchronization\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1220\u001b[0;31m           aggregation=aggregation)\n\u001b[0m\u001b[1;32m   1221\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1222\u001b[0m   def _get_partitioned_variable(self,\n",
      "\u001b[0;32m~/workspace/Tensorflow/.pyenv/lib/python3.7/site-packages/tensorflow/python/ops/variable_scope.py\u001b[0m in \u001b[0;36mget_variable\u001b[0;34m(self, name, shape, dtype, initializer, regularizer, reuse, trainable, collections, caching_device, partitioner, validate_shape, use_resource, custom_getter, constraint, synchronization, aggregation)\u001b[0m\n\u001b[1;32m    528\u001b[0m           function_utils.has_kwargs(custom_getter)):\n\u001b[1;32m    529\u001b[0m         \u001b[0mcustom_getter_kwargs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"constraint\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mconstraint\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 530\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mcustom_getter\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mcustom_getter_kwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    531\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    532\u001b[0m       return _true_getter(\n",
      "\u001b[0;32m~/workspace/Tensorflow/.pyenv/lib/python3.7/site-packages/tensorflow/python/ops/rnn_cell_impl.py\u001b[0m in \u001b[0;36m_rnn_get_variable\u001b[0;34m(self, getter, *args, **kwargs)\u001b[0m\n\u001b[1;32m    235\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    236\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_rnn_get_variable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgetter\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 237\u001b[0;31m     \u001b[0mvariable\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgetter\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    238\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mcontext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexecuting_eagerly\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    239\u001b[0m       \u001b[0mtrainable\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mvariable\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_trainable\u001b[0m  \u001b[0;31m# pylint: disable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/workspace/Tensorflow/.pyenv/lib/python3.7/site-packages/tensorflow/python/ops/variable_scope.py\u001b[0m in \u001b[0;36m_true_getter\u001b[0;34m(name, shape, dtype, initializer, regularizer, reuse, trainable, collections, caching_device, partitioner, validate_shape, use_resource, constraint, synchronization, aggregation)\u001b[0m\n\u001b[1;32m    497\u001b[0m           \u001b[0mconstraint\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mconstraint\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    498\u001b[0m           \u001b[0msynchronization\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msynchronization\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 499\u001b[0;31m           aggregation=aggregation)\n\u001b[0m\u001b[1;32m    500\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    501\u001b[0m     \u001b[0;31m# Set trainable value based on synchronization value.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/workspace/Tensorflow/.pyenv/lib/python3.7/site-packages/tensorflow/python/ops/variable_scope.py\u001b[0m in \u001b[0;36m_get_single_variable\u001b[0;34m(self, name, shape, dtype, initializer, regularizer, partition_info, reuse, trainable, collections, caching_device, validate_shape, use_resource, constraint, synchronization, aggregation)\u001b[0m\n\u001b[1;32m    851\u001b[0m         raise ValueError(\"Trying to share variable %s, but specified shape %s\"\n\u001b[1;32m    852\u001b[0m                          \" and found shape %s.\" % (name, shape,\n\u001b[0;32m--> 853\u001b[0;31m                                                    found_var.get_shape()))\n\u001b[0m\u001b[1;32m    854\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_compatible_with\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfound_var\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    855\u001b[0m         \u001b[0mdtype_str\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: Trying to share variable lstm-words-phonems/rnn/multi_rnn_cell/cell_0/lstm_cell/kernel, but specified shape (19, 64) and found shape (332, 128)."
     ]
    }
   ],
   "source": [
    "rnn = nn.MultiLayerRNN_more_embeddings(name=\"lstm-words-phonems\")\n",
    "rnn.build(num_LSTM_layers, hidden_layer_size, VOCAB_SIZE, TIMESTEPS, embedding_dimension,\n",
    "          num_LSTM_layers_phonem, hidden_layer_size_phonem, VOCAB_SIZE_PHONEM, TIMESTEPS_PHONEM, embedding_dimension_phonem,\n",
    "          l2_reg=0.0)\n",
    "\n",
    "tr.train_model_more_embeddings(rnn, data, data_phonem, labels,\n",
    "                               sampler, epochs=EPOCHS, batch_size=BATCH_SIZE, \n",
    "                               embedding_matrix=word_embedding_matrix,\n",
    "                               embedding_matrix_phonem=phonem_embedding_matrix,\n",
    "                               log_dir=\"logs/10.2-glove-phonem\", retrain=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Now we are able to sample from any given seed into the next function and get new generated rap lyrics\n",
    "### (As long the words of the seed are known words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "decoder.temperature = 0.7\n",
    "\n",
    "sampler = lambda trainable, seed_text: tr.sample( seed_text, trainable, encoder, decoder, length=50)\n",
    "sampler(rnn_words, \"killing people left and right \\n use a gun cool homie \\n that is right\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}