Poem LSTM.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "import argparse\n",
    "import torch\n",
    "import torch.utils.data\n",
    "from torch import nn, optim\n",
    "from torch.nn import functional as F\n",
    "from torchvision import datasets, transforms\n",
    "from torchvision.utils import save_image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "glove.6B.100d.txt    glove.6B.300d.txt    glove.840B.300d.txt\r\n",
      "glove.6B.200d.txt    glove.6B.50d.txt\r\n"
     ]
    }
   ],
   "source": [
    "ls glove"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2195884\n",
      ".\n"
     ]
    }
   ],
   "source": [
    "# prepare glove vectors\n",
    "words = set()\n",
    "word_to_vec = {}\n",
    "word_to_id = {}\n",
    "id_to_word = {}\n",
    "glove_dim = 300\n",
    "glove_filename = \"./glove/glove.840B.300d.txt\" # \"./glove/glove.6B.300d.txt\"\n",
    "with open(glove_filename) as file:\n",
    "    count = 0\n",
    "    for line in file:\n",
    "        tokens = line.split()\n",
    "        word, vals = tokens[0], tokens[-glove_dim:]\n",
    "        if word not in words:\n",
    "            words.add(word)\n",
    "            word_to_vec[word] = torch.Tensor([float(val) for val in vals])\n",
    "            word_to_id[word] = count\n",
    "            id_to_word[count] = word\n",
    "            count += 1\n",
    "print(len(words))\n",
    "print(id_to_word[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_embedding(words, id_to_word, word_to_vec, dim):\n",
    "    weights = torch.stack([word_to_vec[id_to_word[i]] for i in range(len(words))])\n",
    "    embed = nn.Embedding.from_pretrained(weights)\n",
    "    return embed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[-0.0828,  0.6720, -0.1499, -0.0650,  0.0565,  0.4023,  0.0028, -0.3311,\n",
      "         -0.3069,  2.0817,  0.0318,  0.0136,  0.3027,  0.0071, -0.5819, -0.2774,\n",
      "         -0.0623,  1.1451, -0.2423,  0.1235, -0.1224,  0.3315, -0.0062, -0.3054,\n",
      "         -0.1306, -0.0546,  0.0371, -0.0706,  0.5893, -0.3038,  0.2898, -0.1465,\n",
      "         -0.2705,  0.3716,  0.3203, -0.2912,  0.0052, -0.1321, -0.0527,  0.0873,\n",
      "         -0.2667, -0.1690,  0.0152, -0.0084, -0.1487,  0.2341, -0.2072, -0.0914,\n",
      "          0.4008, -0.1722,  0.1814,  0.3759, -0.2868,  0.3729, -0.1619,  0.1801,\n",
      "          0.3032, -0.1322,  0.1835,  0.0958,  0.0949,  0.0083,  0.1176,  0.3405,\n",
      "          0.0368, -0.2908,  0.0583, -0.0278,  0.0829,  0.1862, -0.0315,  0.2799,\n",
      "         -0.0744, -0.1376, -0.2187,  0.1814,  0.0409, -0.1130,  0.2411,  0.3657,\n",
      "         -0.2752, -0.0568,  0.3487,  0.0119,  0.1452, -0.7139,  0.4850,  0.1481,\n",
      "          0.6229,  0.2060,  0.5838, -0.1344,  0.4021,  0.1831,  0.2802, -0.4235,\n",
      "         -0.2563,  0.1771, -0.5410,  0.1660, -0.0361,  0.0850, -0.6499,  0.0755,\n",
      "         -0.2883,  0.4063, -0.2802,  0.0941,  0.3241,  0.2844, -0.2634,  0.1155,\n",
      "          0.0719, -0.4721, -0.1837, -0.3471,  0.2996, -0.6651,  0.0025, -0.4233,\n",
      "          0.2751,  0.3601,  0.1631,  0.2396, -0.0592,  0.3261,  0.2056,  0.0387,\n",
      "         -0.0458,  0.0898,  0.4315, -0.1595,  0.0853, -0.2657, -0.1500,  0.0843,\n",
      "         -0.1671, -0.4300,  0.0608,  0.1312, -0.2411,  0.6655,  0.4453, -0.1802,\n",
      "         -0.1392,  0.5625,  0.2146, -0.4644, -0.0122,  0.0300, -0.0511, -0.2014,\n",
      "          0.8079,  0.4738, -0.0576,  0.4622,  0.1608, -0.2095, -0.0545,  0.1557,\n",
      "         -0.1371,  0.1297, -0.0119, -0.0034, -0.1359, -0.0807,  0.2007,  0.0541,\n",
      "          0.0468,  0.0595,  0.0463,  0.1775, -0.3109,  0.2812, -0.2436,  0.0853,\n",
      "         -0.2101, -0.1947,  0.0027, -0.4634,  0.1479, -0.3152, -0.0659,  0.0361,\n",
      "          0.4290, -0.3376,  0.1643,  0.3257, -0.0504, -0.0543,  0.2407,  0.4192,\n",
      "          0.1301, -0.1717, -0.3781, -0.2309, -0.0195, -0.2929, -0.3082,  0.3030,\n",
      "         -0.2266,  0.0816, -0.1852, -0.2141,  0.4062, -0.2897,  0.0742, -0.1779,\n",
      "          0.2860, -0.0396, -0.2339, -0.3605, -0.0675, -0.0911,  0.2344, -0.0041,\n",
      "          0.0032,  0.0072,  0.0087,  0.2161,  0.0499,  0.3558,  0.1375,  0.0734,\n",
      "          0.1417,  0.2412, -0.0133,  0.1561,  0.0834,  0.0881, -0.0194,  0.4379,\n",
      "          0.0840,  0.4531, -0.5049, -0.1086, -0.2527, -0.1825,  0.2044,  0.1332,\n",
      "          0.1294,  0.0506, -0.1561, -0.3954,  0.1254,  0.2488, -0.1927, -0.3185,\n",
      "         -0.1272,  0.4341,  0.3118, -0.0041, -0.2094, -0.0800,  0.1161, -0.0508,\n",
      "          0.0153, -0.2803, -0.1249,  0.2359,  0.2339, -0.1402,  0.0285,  0.5692,\n",
      "         -0.1649, -0.0364,  0.0101, -0.1711, -0.0426,  0.0450, -0.4393, -0.2614,\n",
      "          0.3009, -0.0608, -0.4531, -0.1908, -0.2029,  0.2769, -0.0609,  0.1194,\n",
      "          0.6221, -0.1934,  0.4785, -0.3011,  0.0594,  0.0749,  0.0611, -0.4662,\n",
      "          0.4005, -0.1910, -0.1433,  0.0183, -0.1864,  0.2071, -0.3560,  0.0534,\n",
      "         -0.0508, -0.1918, -0.3785, -0.0659]])\n"
     ]
    }
   ],
   "source": [
    "embed = make_embedding(['cat', 'dog'], {0:'cat', 1:'dog'}, \n",
    "                       {'cat':torch.FloatTensor([1,1]), 'dog':torch.FloatTensor([2,2])}, 2)\n",
    "glove_embed = make_embedding(words, id_to_word, word_to_vec, glove_dim)\n",
    "print(glove_embed(torch.LongTensor([0])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Poem_LSTM(nn.Module):\n",
    "    def __init__(self, embed, hidden_dim):\n",
    "        super(Poem_LSTM, self).__init__()\n",
    "        self.hidden_dim = hidden_dim\n",
    "        self.embed = embed\n",
    "        self.embed_dim = embed.embedding_dim\n",
    "        self.vocab_size = embed.num_embeddings\n",
    "        self.lstm = nn.LSTM(self.embed_dim, hidden_dim)\n",
    "        self.hidden2tag = nn.Linear(hidden_dim, self.vocab_size)\n",
    "        self.hidden = self.init_hidden()\n",
    "\n",
    "    def init_hidden(self):\n",
    "        return (torch.zeros(1, 1, self.hidden_dim),\n",
    "                torch.zeros(1, 1, self.hidden_dim))\n",
    "\n",
    "    def forward(self, sentence):\n",
    "        embeds = self.embed(sentence)\n",
    "        lstm_out, self.hidden = self.lstm(embeds.view(len(sentence), 1, -1), self.hidden)\n",
    "        tag_space = self.hidden2tag(lstm_out.view(len(sentence), -1))\n",
    "        tag_scores = F.log_softmax(tag_space, dim=1)\n",
    "        return tag_scores\n",
    "    \n",
    "def maxword(id_to_word, tag_scores):\n",
    "    return id_to_word[torch.argmax(tag_scores)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2195884\n",
      "1631776\n",
      "credit-linked\n"
     ]
    }
   ],
   "source": [
    "hidden_dim = 100\n",
    "lstm = Poem_LSTM(glove_embed, hidden_dim)\n",
    "sentence = \"This is a test\"\n",
    "sentence_input = torch.LongTensor([word_to_id[word.lower()] for word in sentence.split()])\n",
    "scores = lstm.forward(sentence_input)\n",
    "word_id = torch.argmax(scores[-1])\n",
    "print(len(scores[0]))\n",
    "print(word_id.item())\n",
    "print(id_to_word[word_id.item()])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import spacy\n",
    "\n",
    "def load_data(vocab, poems, limit):\n",
    "    nlp = spacy.load('en')\n",
    "    line_data = []\n",
    "    scraps = {}\n",
    "    count = 0\n",
    "    for poem in poems:\n",
    "        try:\n",
    "            for line in poem.lower().split('\\n'):\n",
    "                if len(line) == 0:\n",
    "                    continue\n",
    "                tokens = nlp(line)\n",
    "                include = True\n",
    "                bad_words = []\n",
    "                for token in tokens:\n",
    "                    if token.text not in vocab:\n",
    "                        bad_words.append(token.text)\n",
    "                        include = False\n",
    "                if include: line_data.append(line)\n",
    "                else: scraps[line] = bad_words\n",
    "        except:\n",
    "            continue\n",
    "        if count == limit:\n",
    "            break\n",
    "        count += 1\n",
    "    return line_data, scraps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~\n"
     ]
    }
   ],
   "source": [
    "import string\n",
    "print(string.punctuation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2 0\n",
      "{}\n"
     ]
    }
   ],
   "source": [
    "model = Poem_LSTM(glove_embed, hidden_dim)\n",
    "loss_function = nn.NLLLoss()\n",
    "optimizer = optim.Adam(model.parameters(), lr=1e-2)\n",
    "\n",
    "def prepare_training_data(limit=-1):\n",
    "    filename = \"./clean_poems.csv\"\n",
    "    data = pd.read_csv(filename)\n",
    "    #poems = data.iloc[:,3]\n",
    "    poems = load_data(words, data, limit)\n",
    "    return poems\n",
    "    \n",
    "def sentence_to_vector(s, word_to_vec):\n",
    "    ids = [word_to_vec[w] for w in s]\n",
    "    return torch.tensor(ids, dtype=torch.long)\n",
    "\n",
    "def sentence_one_hot(s, word_to_vec):\n",
    "    one_hot = torch.zeros([len(s), len(word_to_vec)], dtype=torch.long)\n",
    "    for i, w in enumerate(s):\n",
    "        one_hot[i, word_to_vec[w]] = 1\n",
    "    return one_hot\n",
    "\n",
    "train_data, scraps = prepare_training_data(limit=-1)\n",
    "print(len(train_data), len(scraps))\n",
    "print(scraps)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "nlp = spacy.load('en')\n",
    "filename = \"clean_poems.csv\"\n",
    "poems = pd.read_csv(filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "339\n",
      "85\n"
     ]
    }
   ],
   "source": [
    "good_lines = []\n",
    "bad_lines  = []\n",
    "\n",
    "def valid_line(line, words):\n",
    "    tokens = nlp(line)\n",
    "    count = 0\n",
    "    for token in tokens:\n",
    "        w = token.text\n",
    "        if w.isalnum():\n",
    "            if w not in words:\n",
    "                return False\n",
    "            count += 1\n",
    "    if count < 2:\n",
    "        return False\n",
    "    return True\n",
    "            \n",
    "for poem in poems.iloc[:10,1]:\n",
    "    lines = poem.split('\\n')\n",
    "    for line in lines:\n",
    "        if valid_line(line, words): good_lines.append(line)\n",
    "        else: bad_lines.append(line)\n",
    "print(len(good_lines))\n",
    "print(len(bad_lines))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Philosophic', '', 'and discrete—a mirror come unsilvered,', '', '', '', 'WINTER', '', 'SUMMER', '', 'STORES', '', 'WRITING', '', 'TODAY', '', '', '', 'RECIPE', 'flour.', '', '', '', '', 'HAPPINESS', '', 'MONEY', '', '', 'EDWARD', '', 'LAKE', '', '', '', 'POTATOES', '', 'MOTHER', '', '', 'sand.', '', 'TODAY', '', '', 'ALASKA', '', '', 'LOYALTY', '', '', 'touch?', '', 'COMPANY', 'but', '', 'lace', 'She', 'betting quarters, sidewheelers and straight thoroughs,', 'look at these whores these onehundreddollar whores', '', '', '', '', '', 'hard-wired giammai to dare say so. So what moved him to not-say', 'have to start to agree. The verbness of it impropriety (eyes glob up the', \"three little words aren't meant as saying. An icy drink in stormlight. A\", 'the centuried moon rose above dinnermint stone; many men contin-', 'ued  talking; a woman lifted her sarsenet skirt, peed on green lilies and,', '', 'Soul, self; come, poor Jackself, I do advise', 'Betweenpie mountains — lights a lovely mile.', '', '', '', '', '', '', '', '', '', '', '', '']\n"
     ]
    }
   ],
   "source": [
    "print(bad_lines)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['in its complex, ovoid emptiness,', 'a skillful pundit coined it as a sort', 'of stopgap doorstop for those']\n"
     ]
    }
   ],
   "source": [
    "print(good_lines[:3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}