06-multi_layer_embedding.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 06 - Embedded Multi Layer LSTM on our preprocessed text\n",
    "\n",
    "Let's feed the generated data from the previous notebook into our LSTM from chapter 4!\n",
    "\n",
    "This time we load our rap text from the **prepped** directory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tools.architectures as nn\n",
    "import tools.processing as pre\n",
    "import tools.training as tr\n",
    "\n",
    "text = pre.get_text(\"data/prepped/clean2_pac.txt\")[:60000]\n",
    "vocab = pre.Vocabulary(text)\n",
    "\n",
    "text2 = pre.get_text(\"data/cleaned-rap-lyrics/clean2_pac_.txt\")[:60000]\n",
    "vocab2 = pre.Vocabulary(text2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### The prepared text should have reduced number of words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vocab size in prepped text: \t1938\n",
      "vocab size in previous text: \t2181\n"
     ]
    }
   ],
   "source": [
    "print( f\"vocab size in prepped text: \\t{len(vocab.index2word)}\")\n",
    "print( f\"vocab size in previous text: \\t{len(vocab2.index2word)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(['as', 'real', 'as', 'it', 'seems', 'the', 'american', 'dream', ';', 'is', 'not', 'nothing', 'but', 'another', 'calculated', 'schemes', ';', 'to', 'get', 'us'], 'locked'), (['real', 'as', 'it', 'seems', 'the', 'american', 'dream', ';', 'is', 'not', 'nothing', 'but', 'another', 'calculated', 'schemes', ';', 'to', 'get', 'us', 'locked'], 'up'), (['as', 'it', 'seems', 'the', 'american', 'dream', ';', 'is', 'not', 'nothing', 'but', 'another', 'calculated', 'schemes', ';', 'to', 'get', 'us', 'locked', 'up'], 'shot'), (['it', 'seems', 'the', 'american', 'dream', ';', 'is', 'not', 'nothing', 'but', 'another', 'calculated', 'schemes', ';', 'to', 'get', 'us', 'locked', 'up', 'shot'], 'up'), (['seems', 'the', 'american', 'dream', ';', 'is', 'not', 'nothing', 'but', 'another', 'calculated', 'schemes', ';', 'to', 'get', 'us', 'locked', 'up', 'shot', 'up'], 'back')]\n"
     ]
    }
   ],
   "source": [
    "TIMESTEPS = 20\n",
    "\n",
    "vocab = pre.Vocabulary(text)\n",
    "\n",
    "tokens = text.split(\" \")[:-1]\n",
    "\n",
    "str_data, str_labels = pre.create_data_label_pairs(tokens, TIMESTEPS)\n",
    "\n",
    "print( list( zip(str_data, str_labels) )[:5] )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "encoder = tr.IndexWordEncoder(\"Index-Word-Encoding\", vocab.word2index)\n",
    "decoder = tr.OneHotWordDecoder(\"1-Hot-Word-Decoding\", vocab.index2word, temperature=0.5)\n",
    "\n",
    "data = encoder.encode( str_data )\n",
    "labels = encoder.encode_labels( str_labels )\n",
    "\n",
    "my_seed = \" \".join(str_data[0]).replace(\"\\\\n\", \"\\n\")\n",
    "\n",
    "del str_data, str_labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Building model from scratch! \n",
      " Saving into: 'logs/06/testing'\n",
      "\n",
      "\n",
      "Epoch 1/15\n",
      "Loss:    \t 5.391435623168945\n",
      "Accuracy:\t 15.765629768371582\n",
      "------Sampling----------\n",
      "seed: \n",
      "as real as it seems the american dream ; is not nothing but another calculated schemes ; to get us\n",
      "-\n",
      "result: \n",
      "as real as it seems the american dream ; is not nothing but another calculated schemes ; to get us the ; i am i am now ; i am the on a nigga ; and i am the ; i am the the ; i am the to i am the am the when ; he is ; i am and like and the to ; the monday the\n",
      "\n",
      "\n",
      "Epoch 2/15\n",
      "Loss:    \t 4.9652838706970215\n",
      "Accuracy:\t 18.838720321655273\n",
      "------Sampling----------\n",
      "seed: \n",
      "as real as it seems the american dream ; is not nothing but another calculated schemes ; to get us\n",
      "-\n",
      "result: \n",
      "as real as it seems the american dream ; is not nothing but another calculated schemes ; to get us to the ready ; i am a smooth america ; but i do not the nigga ; so i am it is a police ; i do not a to get ; i am the facts ; the nigga i am to the keeping and ; i am the other\n",
      "\n",
      "\n",
      "Epoch 3/15\n",
      "Loss:    \t 4.455949306488037\n",
      "Accuracy:\t 23.11235237121582\n",
      "------Sampling----------\n",
      "seed: \n",
      "as real as it seems the american dream ; is not nothing but another calculated schemes ; to get us\n",
      "-\n",
      "result: \n",
      "as real as it seems the american dream ; is not nothing but another calculated schemes ; to get us i get up ; i am the streets for the clip ; i do not the government is not trying punk ; i am a play a like a soulja ; and fuck you ; you all you wanted to be a soulja ; i had a fucking i will\n",
      "\n",
      "\n",
      "Epoch 4/15\n",
      "Loss:    \t 3.8681044578552246\n",
      "Accuracy:\t 28.488370895385742\n",
      "------Sampling----------\n",
      "seed: \n",
      "as real as it seems the american dream ; is not nothing but another calculated schemes ; to get us\n",
      "-\n",
      "result: \n",
      "as real as it seems the american dream ; is not nothing but another calculated schemes ; to get us with the block of the shit ; i am the government the constitution is a for on the life ; he is a nigga nigga nigga is not the shit ; the nigga is the last thing ya need when ya need when ya see me ; i am a\n",
      "\n",
      "\n",
      "Epoch 5/15\n",
      "Loss:    \t 3.3536202907562256\n",
      "Accuracy:\t 35.45756530761719\n",
      "------Sampling----------\n",
      "seed: \n",
      "as real as it seems the american dream ; is not nothing but another calculated schemes ; to get us\n",
      "-\n",
      "result: \n",
      "as real as it seems the american dream ; is not nothing but another calculated schemes ; to get us up to shake up and up up up back back back in the chains ; did not give the streets that we will be beat ; but i am a whole in my gun casanova ; i am a cop in the streets on to jack us ; i am\n",
      "\n",
      "\n",
      "Epoch 6/15\n",
      "Loss:    \t 2.92168927192688\n",
      "Accuracy:\t 40.28239440917969\n",
      "------Sampling----------\n",
      "seed: \n",
      "as real as it seems the american dream ; is not nothing but another calculated schemes ; to get us\n",
      "-\n",
      "result: \n",
      "as real as it seems the american dream ; is not nothing but another calculated schemes ; to get us locked shot down ; i am the feeling of the gun of the clock ; i am the cop cop senseless because he is a lunatic ; thought you fuck you will be a soulja like me ; all you wanted to be a soulja like me ; all you\n",
      "\n",
      "\n",
      "Epoch 7/15\n",
      "Loss:    \t 2.514615058898926\n",
      "Accuracy:\t 50.3171272277832\n",
      "------Sampling----------\n",
      "seed: \n",
      "as real as it seems the american dream ; is not nothing but another calculated schemes ; to get us\n",
      "-\n",
      "result: \n",
      "as real as it seems the american dream ; is not nothing but another calculated schemes ; to get us locked up shot up up back chains ; to deny us up away to deny us up back ; lincoln kept up to the future rob our names ; kept my history ; i had enough of the lies you told told promise me emancipation ; i am the situation\n",
      "\n",
      "\n",
      "Epoch 8/15\n",
      "Loss:    \t 1.9183495044708252\n",
      "Accuracy:\t 61.33343505859375\n",
      "------Sampling----------\n",
      "seed: \n",
      "as real as it seems the american dream ; is not nothing but another calculated schemes ; to get us\n",
      "-\n",
      "result: \n",
      "as real as it seems the american dream ; is not nothing but another calculated schemes ; to get us locked up shot up back ; but to deny us your shit up your hat ; i am the cops on the cops and you can not me ; they are like a lesson of wisdom ; but when i am through the streets with a motherfucking piss ; now\n",
      "\n",
      "\n",
      "Epoch 9/15\n",
      "Loss:    \t 1.4627411365509033\n",
      "Accuracy:\t 71.12654876708984\n",
      "------Sampling----------\n",
      "seed: \n",
      "as real as it seems the american dream ; is not nothing but another calculated schemes ; to get us\n",
      "-\n",
      "result: \n",
      "as real as it seems the american dream ; is not nothing but another calculated schemes ; to get us locked up shot up back in chains ; to deny us of the future rob our names ; kept my history of mystery but now i see ; now the american dream was meant for me ; because lady liberty is a hypocrite she lied to me ; promised me\n",
      "\n",
      "\n",
      "Epoch 10/15\n",
      "Loss:    \t 1.2208309173583984\n",
      "Accuracy:\t 75.30957794189453\n",
      "------Sampling----------\n",
      "seed: \n",
      "as real as it seems the american dream ; is not nothing but another calculated schemes ; to get us\n",
      "-\n",
      "result: \n",
      "as real as it seems the american dream ; is not nothing but another calculated schemes ; to get us locked up shot up shot down ; but i am a cop down who is the jackass ; i am the cops on the streets ; they say like a gun of dough but they will pull ; and i will never let them the ak ; i am the\n",
      "\n",
      "\n",
      "Epoch 11/15\n",
      "Loss:    \t 0.9273813366889954\n",
      "Accuracy:\t 83.4642105102539\n",
      "------Sampling----------\n",
      "seed: \n",
      "as real as it seems the american dream ; is not nothing but another calculated schemes ; to get us\n",
      "-\n",
      "result: \n",
      "as real as it seems the american dream ; is not nothing but another calculated schemes ; to get us locked up shot up back in chains ; to deny us of the future rob our names ; kept my history of mystery but now i see the shit ; the american dream was not meant for me ; because lady liberty is a hypocrite she lied to me ;\n",
      "\n",
      "\n",
      "Epoch 12/15\n",
      "Loss:    \t 0.6680854558944702\n",
      "Accuracy:\t 91.22621154785156\n",
      "------Sampling----------\n",
      "seed: \n",
      "as real as it seems the american dream ; is not nothing but another calculated schemes ; to get us\n",
      "-\n",
      "result: \n",
      "as real as it seems the american dream ; is not nothing but another calculated schemes ; to get us locked up shot up back in chains ; of the future rob our names ; kept my history of mystery but now i see ; the american dream was not meant for me ; because lady liberty is a hypocrite she lied to me ; promised me freedom education equality\n",
      "\n",
      "\n",
      "Epoch 13/15\n",
      "Loss:    \t 0.5235882997512817\n",
      "Accuracy:\t 93.46873474121094\n",
      "------Sampling----------\n",
      "seed: \n",
      "as real as it seems the american dream ; is not nothing but another calculated schemes ; to get us\n",
      "-\n",
      "result: \n",
      "as real as it seems the american dream ; is not nothing but another calculated schemes ; to get us locked up shot up back in chains ; to deny us of the future rob our names ; kept my history of mystery but now i see ; the american dream was not meant for me ; because lady liberty is a hypocrite she lied to me ; promised me\n",
      "\n",
      "\n",
      "Epoch 14/15\n",
      "Loss:    \t 0.3762085735797882\n",
      "Accuracy:\t 96.37571716308594\n",
      "------Sampling----------\n",
      "seed: \n",
      "as real as it seems the american dream ; is not nothing but another calculated schemes ; to get us\n",
      "-\n",
      "result: \n",
      "as real as it seems the american dream ; is not nothing but another calculated schemes ; to get us locked up shot up back in chains ; to deny us of the future rob our names ; kept my history of mystery but now i see ; the american dream was not meant for me ; because lady liberty is a hypocrite she lied to me ; promised me\n",
      "\n",
      "\n",
      "Epoch 15/15\n",
      "Loss:    \t 0.31029078364372253\n",
      "Accuracy:\t 97.07037353515625\n",
      "------Sampling----------\n",
      "seed: \n",
      "as real as it seems the american dream ; is not nothing but another calculated schemes ; to get us\n",
      "-\n",
      "result: \n",
      "as real as it seems the american dream ; is not nothing but another calculated schemes ; to get us locked up shot up back in chains ; to deny us of the future rob ; our names ; strictly dope in the house ; we was not wait to taste each ; but i am looking for a move as bad ; as sure as if i am bad\n"
     ]
    }
   ],
   "source": [
    "HIDDEN_LAYER_SIZE = 512\n",
    "VOCAB_SIZE = len(vocab.word2index)\n",
    "\n",
    "EPOCHS = 15\n",
    "BATCH_SIZE = 256\n",
    "\n",
    "EMBEDDING_SIZE = 256\n",
    "\n",
    "rnn = nn.EmbeddedSingleLayerRNN(name = \"multi-2pac\")\n",
    "rnn.build(HIDDEN_LAYER_SIZE, VOCAB_SIZE, EMBEDDING_SIZE, TIMESTEPS, l2_reg=0.0)\n",
    "\n",
    "sampler = lambda trainable, _: tr.sample( my_seed, trainable, encoder, decoder, length=50)\n",
    "\n",
    "tr.train_model(rnn, data, labels, sampler, epochs=EPOCHS, batch_size=BATCH_SIZE, log_dir=\"logs/06/2pac\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------Sampling----------\n",
      "seed: \n",
      "when i was thirteen ; i had my first love ; there was nobody that could put hands on my baby ; and nobody came between us that could ever come above\n",
      "-\n",
      "result: \n",
      "when i was thirteen ; i had my first love ; there was nobody that could put hands on my baby ; and nobody came between us that could ever come above ; but now i am guilty of falling for his girlfriend ; i know it is wrong but it is not a cop damn ; i am trapped in the hell ; one if you will find out my homies ; it is the nigga that you the nigga ;\n"
     ]
    }
   ],
   "source": [
    "decoder.temperature = 0.8\n",
    "\n",
    "sampler = lambda seed_text: tr.sample( seed_text, rnn, encoder, decoder, length=50)\n",
    "\n",
    "sampler(\"when i was thirteen ; i had my first love ; there was nobody that could put hands on my baby ; and nobody came between us that could ever come above\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}