Friends_Generator.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Friends_Generator.ipynb",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/LauJohansson/DeepLearning_NLP_Friends/blob/master/Friends_Generator.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "O2rMr8O2cFut",
        "colab_type": "text"
      },
      "source": [
        "#How to use"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5IyCeet2cKEX",
        "colab_type": "text"
      },
      "source": [
        "**Press \"runtime\" -> \"run all\"**\n",
        "\n",
        "**In the bottom of this notebook, there will pop up input-boxes. Here's an example of what you can give as inputs:**\n",
        "\n",
        "\n",
        "*Please enter the first word of your Friends manuscript (seperate with space): chandler and ross*\n",
        "\n",
        "*Please enter the number of words that your Friends manuscript should contain: 1000*\n",
        "\n",
        "*Choose a number from 1-100 (if 1, then there is no randomness in word prediction): 5*\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SQs3Zti_Xq4A",
        "colab_type": "text"
      },
      "source": [
        "#Code"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wtBQfWMUXw1k",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from argparse import Namespace\n",
        "\n",
        "flags = Namespace(\n",
        "    mode = 'target', #'pretraining' is pretraining and 'target' is fine-tuning of either PTB data or friends data. \n",
        "    name='Friends',\n",
        "    seq_size= 32,                        #Sequence Length\n",
        "    batch_size=40,                       #Batch size\n",
        "    embedding_size=256,                  #Embedding size\n",
        "    lstm_size=512,                       #Hidden nodes size\n",
        "    gradients_norm=0.5,   \n",
        "    #initial_words=['banknote', 'berlitz'],\n",
        "    initial_words_train = [],\n",
        "    initial_words_valid= [],\n",
        "    predict_top_k=5,                    #Choose the k best next_word_prediction, and a random is chosing.\n",
        "    checkpoint_path='checkpoint',\n",
        "    total_epochs=100,                   #Choose number of epochs in training\n",
        "    learning_rate=0.001,                #Choose number of epochs in training\n",
        "    predict_every=1000,\n",
        "    #validation_corpus_size=len(valid_file.split()),\n",
        "    dropconnect_rate=0.4,               #Choose drop connect rate\n",
        "    n_lay=2,                            #Choose number of LSTM layers\n",
        "\n",
        "    #Set variational sequence length on/off\n",
        "    var_seq='Y',                            #Choose if variational sequence length is on/off\n",
        "    var_seq_std=2,                          #Choose std. dev. for norm distribution for var. seq. length ( in moment 1/2 of seq length)\n",
        "\n",
        "\n",
        "    #scheduler parameters\n",
        "    schedule_on='N',                        #Choose if LR-scheduler is on/off\n",
        "    triangular='N',                         #Choose 'Y' to turn on the slanted triangular LR. \n",
        "    cut_fracI=0.2,                          #Choose the fraction of iterations we increase the LR in STL\n",
        "    ratioI=32,                              #Choose how much smaller the lowest LR is from the maximum LR ηmax\n",
        "    nmaxI=0.0,                              #this will be set = learning_rate \n",
        "\n",
        "    #Use same drop-mask for drop-connect\n",
        "    same_drop_lstm='N',                     #Choose 'Y' if drop-connect all should use same mask\n",
        "    \n",
        "    #Dropout on embedding layer\n",
        "    drop_embed=0.5,                         #Choose dropoutrate for embedding dropout        \n",
        "\n",
        "    #Optimizer selection\n",
        "    optim_select='AdamW'                      #Choose between \"AdamW, SGD, ASGD\"\n",
        "\n",
        ")\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IhaW_SmSzWgj",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def get_data_from_file(train_file, batch_size, seq_size):\n",
        "  \n",
        "    text=train_file.split()\n",
        "    # Extend words_notinpretraining to text to get them as a part of the mapping dictionary\n",
        "    #text.extend(words_notinpretraining)\n",
        "    word_counts = Counter(text)\n",
        "    sorted_vocab = sorted(word_counts, key=word_counts.get, reverse=True)\n",
        "    int_to_vocab = {k: w for k, w in enumerate(sorted_vocab)}\n",
        "    vocab_to_int = {w: k for k, w in int_to_vocab.items()}\n",
        "    n_vocab = len(int_to_vocab)\n",
        "\n",
        "    text=train_file.split()\n",
        "\n",
        "    print('Vocabulary size', n_vocab)\n",
        "\n",
        "    int_text = [vocab_to_int[w] for w in text]\n",
        "    num_batches = int(len(int_text) / (seq_size * batch_size))\n",
        "    in_text = int_text[:num_batches * batch_size * seq_size]\n",
        "    out_text = np.zeros_like(in_text)\n",
        "    out_text[:-1] = in_text[1:]\n",
        "    out_text[-1] = in_text[0]\n",
        "    in_text = np.reshape(in_text, (batch_size, -1))\n",
        "    out_text = np.reshape(out_text, (batch_size, -1))\n",
        "    return int_to_vocab, vocab_to_int, n_vocab, in_text, out_text, sorted_vocab"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Agm7zSvWzYoX",
        "colab_type": "code",
        "outputId": "6c4cef28-8679-4ee4-b35b-9a2c6434c649",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "\n",
        "\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "from collections import Counter\n",
        "from urllib.request import urlopen\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
        "\n",
        "data = str(urlopen('https://raw.githubusercontent.com/LauJohansson/DeepLearning_NLP_Friends/master/Data/friends_train.txt').read(),encoding=\"utf-8\")\n",
        "\n",
        "int_to_vocab, vocab_to_int, n_vocab, in_text, out_text,sorted_vocab = get_data_from_file( \n",
        "          data, flags.batch_size, flags.seq_size)\n",
        "      \n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Vocabulary size 11431\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RXoJ0zOClMTe",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class RNNModule(nn.Module):\n",
        "    def __init__(self, n_vocab, seq_size, embedding_size, lstm_size):\n",
        "        super(RNNModule, self).__init__()\n",
        "        self.seq_size = seq_size\n",
        "        self.lstm_size = lstm_size\n",
        "        self.embedding = nn.Embedding(n_vocab, embedding_size)\n",
        "        self.lstm=nn.LSTM(embedding_size,\n",
        "                            lstm_size,\n",
        "                            batch_first=True,num_layers=flags.n_lay)\n",
        "        \n",
        "        self.dense = nn.Linear(lstm_size, n_vocab)\n",
        "        self.drop_out=nn.Dropout(flags.drop_embed)\n",
        "        self.OneMaskOnly=torch.autograd.Variable(torch.Tensor(getattr(self.lstm,self.lstm._all_weights[0][0]).shape[0],\n",
        "                                                        getattr(self.lstm,self.lstm._all_weights[0][0]).shape[1]).uniform_().to(\"cuda\") > flags.dropconnect_rate)\n",
        "\n",
        "\n",
        "    def forward(self, x, prev_state):\n",
        "        embed = self.embedding(x)\n",
        "        embed=self.drop_out(embed)\n",
        "        orig=[]\n",
        "\n",
        "        #Make dropconnect\n",
        "        if self.training:\n",
        "          for i in range( len(self.lstm._all_weights[0])):\n",
        "            name = self.lstm._all_weights[0][i]\n",
        "            if name.find('LSTM.weight_hh_l')!=-1:      \n",
        "              orig.append(getattr(self.lstm,name))\n",
        "            \n",
        "              if flags.same_drop_lstm=='Y':\n",
        "                mask=self.OneMaskOnly\n",
        "              else:\n",
        "                mask=torch.autograd.Variable(torch.Tensor(getattr(self.lstm,name).shape[0],\n",
        "                                                        getattr(self.lstm,name).shape[1]).uniform_().to(\"cuda\") > flags.flags.dropconnect_rate)\n",
        "              setattr(self.lstm,name,torch.nn.Parameter(torch.mul(getattr(self.lstm,name),mask)))\n",
        "              \n",
        "              self.lstm.flatten_parameters()\n",
        "             \n",
        "\n",
        "        #LSTM forward\n",
        "        output, state = self.lstm(embed, prev_state)\n",
        "\n",
        "        #Set hh-weight back to original\n",
        "        if self.training:\n",
        "          a=0\n",
        "          for i in range( len(self.lstm._all_weights[0])):\n",
        "            name = self.lstm._all_weights[0][i]\n",
        "            if name.find('LSTM.weight_hh_l')!=-1:\n",
        "              print(orig)\n",
        "              setattr(self.lstm,name,orig[a])\n",
        "              #self.lstm.weight_hh_l0=orig\n",
        "              self.lstm.flatten_parameters()\n",
        "              a=+1\n",
        "\n",
        "        logits = self.dense(output)\n",
        "\n",
        "        return logits, state\n",
        "       \n",
        "    def zero_state(self, batch_size):\n",
        "        return (torch.zeros(flags.n_lay, batch_size, self.lstm_size),\n",
        "                torch.zeros(flags.n_lay, batch_size, self.lstm_size))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GxcklPI-kB19",
        "colab_type": "code",
        "outputId": "151e985a-c8f3-41a6-9708-189b14266b18",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 119
        }
      },
      "source": [
        "net = RNNModule(n_vocab, flags.seq_size,\n",
        "                    flags.embedding_size, flags.lstm_size)\n",
        "\n",
        "net.load_state_dict(torch.hub.load_state_dict_from_url('https://raw.githubusercontent.com/LauJohansson/DeepLearning_NLP_Friends/master/Data/net_final_online_ToGenerator_v1.pth'))\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "net = net.to(device)\n",
        "\n",
        "net.eval()\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "RNNModule(\n",
              "  (embedding): Embedding(11431, 256)\n",
              "  (lstm): LSTM(256, 512, num_layers=2, batch_first=True)\n",
              "  (dense): Linear(in_features=512, out_features=11431, bias=True)\n",
              "  (drop_out): Dropout(p=0.5, inplace=False)\n",
              ")"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 117
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "G5ucakDfkAJu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def predict(device, net, words, n_vocab, vocab_to_int, int_to_vocab, top_k,manu_length):\n",
        "    #net.eval()\n",
        "\n",
        "    state_h, state_c = net.zero_state(1)\n",
        "    state_h = state_h.to(device)\n",
        "    state_c = state_c.to(device)\n",
        "    for w in words:\n",
        "        ix = torch.tensor([[vocab_to_int[w]]]).to(device)\n",
        "        output, (state_h, state_c) = net(ix, (state_h, state_c))\n",
        "    \n",
        "\n",
        "    _, top_ix = torch.topk(output[0], k=top_k)\n",
        "    choices = top_ix.tolist()\n",
        "\n",
        "    #Removing \"<unk>\" from the predictions\n",
        "    for word in choices[0]:  # iterating on a copy since removing will mess things up\n",
        "      if word==vocab_to_int[\"<unk>\"]:\n",
        "        choices[0].remove(word)\n",
        "\n",
        "    choice = np.random.choice(choices[0]) #A way to avoid always choose like \"and\" \"then\"..... \n",
        "\n",
        "    words.append(int_to_vocab[choice])\n",
        "\n",
        "    for _ in range(manu_length):\n",
        "        ix = torch.tensor([[choice]]).to(device)\n",
        "        output, (state_h, state_c) = net(ix, (state_h, state_c))\n",
        "\n",
        "        _, top_ix = torch.topk(output[0], k=top_k)\n",
        "        choices = top_ix.tolist()\n",
        "        #Removing \"<unk>\" from the predictions\n",
        "        for word in choices[0]:  # iterating on a copy since removing will mess things up\n",
        "          if word==vocab_to_int[\"<unk>\"]:\n",
        "            choices[0].remove(word)\n",
        "         \n",
        "        choice = np.random.choice(choices[0])\n",
        "        words.append(int_to_vocab[choice])\n",
        "\n",
        "    \n",
        "\n",
        "    output= ' '.join(words)#.replace('. ', '.\\n')\n",
        "\n",
        "    output=output.replace('chandler:', '\\nchandler:')\n",
        "    output=output.replace('ross:', '\\nross:')\n",
        "    output=output.replace('joey:', '\\njoey')\n",
        "    output=output.replace('monica:', '\\nmonica:')\n",
        "    output=output.replace('phoebe:', '\\nphoebe:')\n",
        "    output=output.replace('rachel:', '\\nrachel:')\n",
        "\n",
        "    \n",
        "    return output"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SU6yJiC9xCEe",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def get_init_words():\n",
        "  initial_words = input(\"Please enter the first words of your Friends manuscript (seperate with space): \")\n",
        "  inital_words=initial_words.lower()\n",
        "  initial_words=initial_words.split()\n",
        "  initial_words=['[','scene:'] + initial_words\n",
        "  return initial_words"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "f921DXqP10Ix",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def generate_manuscript():\n",
        "  initial_words=get_init_words()\n",
        "\n",
        "  number_of_words = int(input(\"Please enter the number of words that your Friends manuscript should contain: \"))\n",
        "\n",
        "  randomness = int(input(\"Choose a number from 1-100 (recommend 5) (if 1, then there is no randomness in word prediction): \"))\n",
        "\n",
        "  if randomness <1:\n",
        "    randomness=1\n",
        "  if randomness>100:\n",
        "    randomness=100  \n",
        "\n",
        "\n",
        "  #Only print of max 10000 words\n",
        "  if number_of_words>10000:\n",
        "    number_of_words=10000\n",
        "\n",
        "  #Make loop which checks if the entered words are in the vocab\n",
        "  first_loop=1\n",
        "  good_list=0\n",
        "\n",
        "  while good_list==0 or first_loop==1:\n",
        "    first_loop=0\n",
        "    good_list=1\n",
        "    result=all(elem in sorted_vocab for elem in  initial_words)\n",
        "\n",
        "    if not result:\n",
        "      good_list=0\n",
        "    if good_list==0:\n",
        "      print('Some of the words you entered is not a word that is used in Friends')\n",
        "      initial_words=get_init_words()\n",
        "  print('\\n')\n",
        "  print( predict(device, net, initial_words, n_vocab,\n",
        "                            vocab_to_int, int_to_vocab, randomness,number_of_words))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4EXdABCiX0Rm",
        "colab_type": "text"
      },
      "source": [
        "#Generate manuscript:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Hsw82MiX2DTp",
        "colab_type": "code",
        "outputId": "decb8e42-6c1e-4230-846c-ff255e4b5c88",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 989
        }
      },
      "source": [
        "generate_manuscript()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Please enter the first word of your Friends manuscript (seperate with space): rachel and ross\n",
            "Please enter the number of words that your Friends manuscript should contain: 1000\n",
            "Choose a number from 1-100 (recommend 5) (if 1, then there is no randomness in word prediction): 5\n",
            "\n",
            "\n",
            "[ scene: rachel and ross enter with the bathroom , ross enters . and ross are there to find her a cup , and the gang who has to come in . ] \n",
            "phoebe: oh , hey - ho , hey rach ? \n",
            "ross: i just wanted her . \n",
            "rachel: i was thinking of the most of those guys were having the last day . \n",
            "ross: i don't know ! \n",
            "ross: yeah ? \n",
            "monica:: i think i'm going out of this . \n",
            "phoebe: i am . ( to ross and ross are shocked , and rachel and phoebe ) \n",
            "monica:: what ? \n",
            "ross: oh - huh , i'm gonna be in here ! ( to ross again ) you can get it out . \n",
            "phoebe: yeah i am so good . i mean i'm just a good person with her and i know , but y'know if you should just say , you have some kind , but then you have a lot for me and y'know i just want a date to do . \n",
            "ross: well then i'm gonna do anything about you . \n",
            "rachel: yeah i - i just think i would know . \n",
            "rachel: ( laughs , and ross looks out of a little - voice , a - - a lot ! ) [ scene: central , chandler , and phoebe's are returning as joey is still in the chair . joey enters with her . she opens her door with the door and starts reading a magazine and starts wearing the phone on the floor , the door , chandler answers it , then chandler , chandler enters . joey enters the bedroom and runs over and closes his apartment . ] \n",
            "chandler: hey - whoa . \n",
            "chandler: ( to \n",
            "monica: and joey . ross , chandler . \n",
            "monica: ) hey joey and joey . \n",
            "joey oh yeah - ho . oh - ho - ha ! \n",
            "monica:: yeah ? ( joey turns around the bathroom to see the other table is in his chair ) \n",
            "monica:: what ? what ? what is the matter about that i ? ! \n",
            "joey what do they have to do that . \n",
            "monica:: ( looking in front and joey looks at his chest . he looks out on her . but then they start back into their bedroom . chandler has to get out with a big look . ) \n",
            "chandler: what is you gonna get the big date ! ! \n",
            "chandler: oh - ho , it's just my best friend ! \n",
            "joey what - why ? \n",
            "monica:: ( to \n",
            "monica: , joey and joey both laugh , and he is sitting in his room and starts a very passionate look at his face . joey and \n",
            "monica: are still laughing . joey , ross , joey is looking around and stops and chandler and chandler both enter and joey glares around and chandler looks in his room . ) \n",
            "monica:: what are the odds you do . you don't get to go ! \n",
            "chandler: oh my gosh - no ! ! i know what happened ! ! ! \n",
            "chandler: i know , and it would not be weird , but it's a big idea for my own . \n",
            "monica:: oh no , i'm sorry , you're a little bit , i'm a good guy , i am just not going out for you ! i can't believe i am gonna have the rest of you and i'm not ! \n",
            "joey okay ! \n",
            "chandler: i don't care of you and me ! ( pause . joey looks at the window and stops and looks out of his face ) \n",
            "chandler: what ? \n",
            "chandler: well i don't care of that i don't have any more of this , so if you're going out with it . \n",
            "chandler: well , i'm gonna be in there and get a lot to be at the same time , i don't care . ( pause ) i'm sorry i can do anything for him . i just thought you can do anything for me . \n",
            "chandler: oh no . \n",
            "joey no . i can't ! \n",
            "joey no . \n",
            "chandler: oh yeah ? \n",
            "monica:: i mean i'm going in here for you ! \n",
            "chandler: ( entering from his bedroom and the door is sitting around the bathroom and chandler is shocked ) hi , i'm a good guy . \n",
            "joey oh , yeah . ( she goes back over to his room and stops him and he turns around and starts making up and stops him on the couch , and the door starts making him out with her and chandler is wearing her face and the rest . joey is shocked . chandler enters with the duck . chandler enters , carrying his eyes on his chair and runs back and sits in front with him ) \n",
            "monica:: what is this going on , what is that you doing this ? \n",
            "joey what happened ? \n",
            "joey well i just got it ! \n",
            "monica:: oh my god . \n",
            "chandler: i know that was a little good . \n",
            "monica:: yeah i think i can . ( pause and he turns out and starts a huge laugh , joey , joey , who , and joey are laughing at each other for a second , but the chick , and chandler and the girls start to get the one to stop it . chandler starts a big time to the one . joey is in pain and joey has been on a chair to the same side , the rest , the next time and chandler gets out , and joey is looking for him . he tries a lot of it , then he doesn't see the face and then she starts in a few of a face and the one , then he tries it to get a ball , but the rest of his head in , the one of them . the duck is in front by chandler\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}