From 42522693fcbea2e7dc7a5c1270d7bdab8710a50e Mon Sep 17 00:00:00 2001 From: Bryce McWilliams Date: Fri, 10 Jul 2020 19:29:23 +0200 Subject: [PATCH] feat: add nlp 101 --- nlp_101.ipynb | 1085 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 1085 insertions(+) create mode 100644 nlp_101.ipynb diff --git a/nlp_101.ipynb b/nlp_101.ipynb new file mode 100644 index 0000000..fc61bf3 --- /dev/null +++ b/nlp_101.ipynb @@ -0,0 +1,1085 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "name": "nlp_101.ipynb", + "provenance": [], + "authorship_tag": "ABX9TyNjj+II+0ZLdlUhZ9ycFP4B", + "include_colab_link": true + }, + "kernelspec": { + "name": "python3", + "display_name": "Python 3" + } + }, + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "view-in-github", + "colab_type": "text" + }, + "source": [ + "\"Open" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "5dmft22JszFn", + "colab_type": "code", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 34 + }, + "outputId": "80354877-3829-4aff-d362-b3e5102eacc4" + }, + "source": [ + "import nltk\n", + "#nltk.download('movie_reviews')\n", + "from nltk.corpus import movie_reviews\n", + "movie_reviews.words()" + ], + "execution_count": 2, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "['plot', ':', 'two', 'teen', 'couples', 'go', 'to', ...]" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 2 + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "4-B-SrRtrB75", + "colab_type": "text" + }, + "source": [ + "#Data Preprocessing" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "qlJ1EJewrFx1", + "colab_type": "text" + }, + "source": [ + "## 1. Tokenization" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "z0kVzWJmraHy", + "colab_type": "text" + }, + "source": [ + "###1.1 Word" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "KsGROjs7qLOE", + "colab_type": "code", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 51 + }, + "outputId": "aed49aac-8936-47c9-b1b4-3afba2a360e5" + }, + "source": [ + "from nltk.tokenize import word_tokenize\n", + "\n", + "#nltk.download('punkt')\n", + "\n", + "data = \"I like skewl\"\n", + "\n", + "tokenized_text=word_tokenize(data)\n", + "print(tokenized_text)\n", + "print(type(tokenized_text))" + ], + "execution_count": 4, + "outputs": [ + { + "output_type": "stream", + "text": [ + "['I', 'like', 'skewl']\n", + "\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "xozestRYrNIO", + "colab_type": "text" + }, + "source": [ + "\n", + "###1.2 Sentence" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "H6td1iLuqfon", + "colab_type": "code", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 71 + }, + "outputId": "bc299b80-5eb5-4dd7-a00d-acd46c6f2669" + }, + "source": [ + "from nltk.tokenize import sent_tokenize\n", + "\n", + "paragraph=\"\"\"Cake is a form of sweet food made from flour, sugar, and other ingredients, that is usually baked.In their oldest forms, cakes were modifications of bread, but cakes now cover a wide range of preparations that can be simple or elaborate, and that share features with other desserts such as pastries, meringues, custards, and pies.The most commonly used cake ingredients include flour, sugar, eggs, butter or oil or margarine, a liquid, and leavening agents, such as baking soda or baking powder. Common additional ingredients and flavourings include dried, candied, or fresh fruit, nuts, cocoa, and extracts such as vanilla, with numerous substitutions for the primary ingredients.Cakes can also be filled with fruit preserves, nuts or dessert sauces (like pastry cream), iced with buttercream or other icings, and decorated with marzipan, piped borders, or candied fruit.\"\"\"\n", + "\n", + "tokenized_paragraph=sent_tokenize(paragraph)\n", + "print(tokenized_paragraph)\n", + "print(type(tokenized_paragraph))" + ], + "execution_count": 5, + "outputs": [ + { + "output_type": "stream", + "text": [ + "['Cake is a form of sweet food made from flour, sugar, and other ingredients, that is usually baked.In their oldest forms, cakes were modifications of bread, but cakes now cover a wide range of preparations that can be simple or elaborate, and that share features with other desserts such as pastries, meringues, custards, and pies.The most commonly used cake ingredients include flour, sugar, eggs, butter or oil or margarine, a liquid, and leavening agents, such as baking soda or baking powder.', 'Common additional ingredients and flavourings include dried, candied, or fresh fruit, nuts, cocoa, and extracts such as vanilla, with numerous substitutions for the primary ingredients.Cakes can also be filled with fruit preserves, nuts or dessert sauces (like pastry cream), iced with buttercream or other icings, and decorated with marzipan, piped borders, or candied fruit.']\n", + "\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "l4gjpctQrkaY", + "colab_type": "text" + }, + "source": [ + "##2. Punctuation Removal" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "X6NE31DXqs6o", + "colab_type": "code", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 34 + }, + "outputId": "7abe3695-4c28-4213-9ebb-1a703910afa5" + }, + "source": [ + "from nltk.tokenize import RegexpTokenizer\n", + "\n", + "tokenizer = RegexpTokenizer(r'\\w+')\n", + "\n", + "result = tokenizer.tokenize(\"Wow! I am excited to learn science\")\n", + "print(result)" + ], + "execution_count": 7, + "outputs": [ + { + "output_type": "stream", + "text": [ + "['Wow', 'I', 'am', 'excited', 'to', 'learn', 'science']\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "40A-9-eKruLk", + "colab_type": "text" + }, + "source": [ + "##3. Stop Words Removal" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "YfLgTlJ2q53Q", + "colab_type": "code", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 71 + }, + "outputId": "6274393a-9a44-4e76-b8cd-e1e51821a821" + }, + "source": [ + "from nltk.corpus import stopwords\n", + "from nltk.tokenize import word_tokenize\n", + "\n", + "#nltk.download('stopwords')\n", + "\n", + "to_be_removed = set(stopwords.words('english'))\n", + "\n", + "para=\"\"\"Cake is a form of sweet food made from flour, sugar, and other ingredients, that is usually baked.\n", + "In their oldest forms, cakes were modifications of bread, but cakes now cover a wide range of preparations \n", + "that can be simple or elaborate, and that share features with other desserts such as pastries, meringues, custards, \n", + "and pies.\"\"\"\n", + "\n", + "tokenized_para=word_tokenize(para)\n", + "print(tokenized_para)\n", + "\n", + "modified_token_list=[word for word in tokenized_para if not word in to_be_removed]\n", + "print(modified_token_list)" + ], + "execution_count": 8, + "outputs": [ + { + "output_type": "stream", + "text": [ + "['Cake', 'is', 'a', 'form', 'of', 'sweet', 'food', 'made', 'from', 'flour', ',', 'sugar', ',', 'and', 'other', 'ingredients', ',', 'that', 'is', 'usually', 'baked', '.', 'In', 'their', 'oldest', 'forms', ',', 'cakes', 'were', 'modifications', 'of', 'bread', ',', 'but', 'cakes', 'now', 'cover', 'a', 'wide', 'range', 'of', 'preparations', 'that', 'can', 'be', 'simple', 'or', 'elaborate', ',', 'and', 'that', 'share', 'features', 'with', 'other', 'desserts', 'such', 'as', 'pastries', ',', 'meringues', ',', 'custards', ',', 'and', 'pies', '.']\n", + "['Cake', 'form', 'sweet', 'food', 'made', 'flour', ',', 'sugar', ',', 'ingredients', ',', 'usually', 'baked', '.', 'In', 'oldest', 'forms', ',', 'cakes', 'modifications', 'bread', ',', 'cakes', 'cover', 'wide', 'range', 'preparations', 'simple', 'elaborate', ',', 'share', 'features', 'desserts', 'pastries', ',', 'meringues', ',', 'custards', ',', 'pies', '.']\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "GCrBnfsIsLKU", + "colab_type": "text" + }, + "source": [ + "##4. Stemming" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "b1qksx6esSL6", + "colab_type": "text" + }, + "source": [ + "###4.1 Porter Stemmer" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "5z-Y8O1jr0Bm", + "colab_type": "code", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 54 + }, + "outputId": "f3fe6773-276d-4ae1-dcf4-06c794efc619" + }, + "source": [ + "from nltk.stem import PorterStemmer\n", + "from nltk.tokenize import word_tokenize\n", + "\n", + "stemmer = PorterStemmer()\n", + "\n", + "content = \"\"\"Cake is a form of sweet food made from flour, sugar, and other ingredients, that is usually baked.In their oldest forms, cakes were modifications of bread, but cakes now cover a wide range of preparations \n", + "that can be simple or elaborate, and that share features with other desserts such as pastries, meringues, custards, and pies.\"\"\"\n", + "\n", + "tk_content=word_tokenize(content)\n", + "\n", + "stemmed_words = [stemmer.stem(i) for i in tk_content] \n", + "print(stemmed_words)" + ], + "execution_count": 17, + "outputs": [ + { + "output_type": "stream", + "text": [ + "['cake', 'is', 'a', 'form', 'of', 'sweet', 'food', 'made', 'from', 'flour', ',', 'sugar', ',', 'and', 'other', 'ingredi', ',', 'that', 'is', 'usual', 'baked.in', 'their', 'oldest', 'form', ',', 'cake', 'were', 'modif', 'of', 'bread', ',', 'but', 'cake', 'now', 'cover', 'a', 'wide', 'rang', 'of', 'prepar', 'that', 'can', 'be', 'simpl', 'or', 'elabor', ',', 'and', 'that', 'share', 'featur', 'with', 'other', 'dessert', 'such', 'as', 'pastri', ',', 'meringu', ',', 'custard', ',', 'and', 'pie', '.']\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "AUK64-yVseOW", + "colab_type": "text" + }, + "source": [ + "###4.2 Lancaster Stemmer" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "EaJJzeJisW8R", + "colab_type": "code", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 54 + }, + "outputId": "6c3c2064-3944-4e6f-ace9-5b35a89f7c9c" + }, + "source": [ + "from nltk.stem import LancasterStemmer\n", + "from nltk.tokenize import word_tokenize\n", + "\n", + "stemmer = PorterStemmer()\n", + "\n", + "content = \"\"\"Cake is a form of sweet food made from flour, sugar, and other ingredients, that is usually baked.\n", + "In their oldest forms, cakes were modifications of bread, but cakes now cover a wide range of preparations \n", + "that can be simple or elaborate, and that share features with other desserts such as pastries, meringues, custards, \n", + "and pies.\"\"\"\n", + "\n", + "tk_content=word_tokenize(content)\n", + "\n", + "stemmed_words = [stemmer.stem(i) for i in tk_content]\n", + "print(stemmed_words)" + ], + "execution_count": 18, + "outputs": [ + { + "output_type": "stream", + "text": [ + "['cake', 'is', 'a', 'form', 'of', 'sweet', 'food', 'made', 'from', 'flour', ',', 'sugar', ',', 'and', 'other', 'ingredi', ',', 'that', 'is', 'usual', 'bake', '.', 'In', 'their', 'oldest', 'form', ',', 'cake', 'were', 'modif', 'of', 'bread', ',', 'but', 'cake', 'now', 'cover', 'a', 'wide', 'rang', 'of', 'prepar', 'that', 'can', 'be', 'simpl', 'or', 'elabor', ',', 'and', 'that', 'share', 'featur', 'with', 'other', 'dessert', 'such', 'as', 'pastri', ',', 'meringu', ',', 'custard', ',', 'and', 'pie', '.']\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Pkt2nvyLswWD", + "colab_type": "text" + }, + "source": [ + "##5. Lemmentization" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "s3Rfm6W7tBw8", + "colab_type": "text" + }, + "source": [ + "###5.1 WordNet Lemmatizer" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "-awZ_d49sk7U", + "colab_type": "code", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 88 + }, + "outputId": "a95d4c6c-2ee7-4bf7-fce6-48597e9ed2ac" + }, + "source": [ + "from nltk.stem import WordNetLemmatizer\n", + "\n", + "nltk.download('wordnet')\n", + "\n", + "lemmatizer=WordNetLemmatizer()\n", + "\n", + "content = \"\"\"Cake is a form of sweet food made from flour, sugar, and other ingredients, that is usually baked.\n", + "In their oldest forms, cakes were modifications of bread, but cakes now cover a wide range of preparations \n", + "that can be simple or elaborate, and that share features with other desserts such as pastries, meringues, custards, \n", + "and pies.\"\"\"\n", + "\n", + "tk_content=word_tokenize(content)\n", + "\n", + "lemmatized_words = [lemmatizer.lemmatize(i) for i in tk_content] \n", + "print(lemmatized_words)" + ], + "execution_count": 19, + "outputs": [ + { + "output_type": "stream", + "text": [ + "[nltk_data] Downloading package wordnet to /root/nltk_data...\n", + "[nltk_data] Package wordnet is already up-to-date!\n", + "['Cake', 'is', 'a', 'form', 'of', 'sweet', 'food', 'made', 'from', 'flour', ',', 'sugar', ',', 'and', 'other', 'ingredient', ',', 'that', 'is', 'usually', 'baked', '.', 'In', 'their', 'oldest', 'form', ',', 'cake', 'were', 'modification', 'of', 'bread', ',', 'but', 'cake', 'now', 'cover', 'a', 'wide', 'range', 'of', 'preparation', 'that', 'can', 'be', 'simple', 'or', 'elaborate', ',', 'and', 'that', 'share', 'feature', 'with', 'other', 'dessert', 'such', 'a', 'pastry', ',', 'meringue', ',', 'custard', ',', 'and', 'pie', '.']\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "6t864u2wtHlS", + "colab_type": "text" + }, + "source": [ + "## 6. POS Tagging" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "2dXnFzZ0s8AN", + "colab_type": "code", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 54 + }, + "outputId": "48e664ae-2e28-4220-9069-fd8855569cb1" + }, + "source": [ + "from nltk.tokenize import sent_tokenize, word_tokenize\n", + "\n", + "#nltk.download('averaged_perceptron_tagger')\n", + "#nltk.download('universal_tagset')\n", + "\n", + "content = \"\"\"Cake is a form of sweet food made from flour, sugar, and other ingredients, that is usually baked.\n", + "In their oldest forms, cakes were modifications of bread, but cakes now cover a wide range of preparations \n", + "that can be simple or elaborate, and that share features with other desserts such as pastries, meringues, custards, \n", + "and pies.\"\"\"\n", + "\n", + "words = [word_tokenize(i) for i in sent_tokenize(content)]\n", + "\n", + "pos_tag = [nltk.pos_tag(i,tagset=\"universal\") for i in words]\n", + "print(pos_tag)" + ], + "execution_count": 20, + "outputs": [ + { + "output_type": "stream", + "text": [ + "[[('Cake', 'NOUN'), ('is', 'VERB'), ('a', 'DET'), ('form', 'NOUN'), ('of', 'ADP'), ('sweet', 'ADJ'), ('food', 'NOUN'), ('made', 'VERB'), ('from', 'ADP'), ('flour', 'NOUN'), (',', '.'), ('sugar', 'NOUN'), (',', '.'), ('and', 'CONJ'), ('other', 'ADJ'), ('ingredients', 'NOUN'), (',', '.'), ('that', 'DET'), ('is', 'VERB'), ('usually', 'ADV'), ('baked', 'VERB'), ('.', '.')], [('In', 'ADP'), ('their', 'PRON'), ('oldest', 'ADJ'), ('forms', 'NOUN'), (',', '.'), ('cakes', 'NOUN'), ('were', 'VERB'), ('modifications', 'NOUN'), ('of', 'ADP'), ('bread', 'NOUN'), (',', '.'), ('but', 'CONJ'), ('cakes', 'NOUN'), ('now', 'ADV'), ('cover', 'VERB'), ('a', 'DET'), ('wide', 'ADJ'), ('range', 'NOUN'), ('of', 'ADP'), ('preparations', 'NOUN'), ('that', 'DET'), ('can', 'VERB'), ('be', 'VERB'), ('simple', 'ADJ'), ('or', 'CONJ'), ('elaborate', 'ADJ'), (',', '.'), ('and', 'CONJ'), ('that', 'ADP'), ('share', 'NOUN'), ('features', 'NOUN'), ('with', 'ADP'), ('other', 'ADJ'), ('desserts', 'NOUN'), ('such', 'ADJ'), ('as', 'ADP'), ('pastries', 'NOUN'), (',', '.'), ('meringues', 'NOUN'), (',', '.'), ('custards', 'NOUN'), (',', '.'), ('and', 'CONJ'), ('pies', 'NOUN'), ('.', '.')]]\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "C6AYSxBQtj4Y", + "colab_type": "text" + }, + "source": [ + "##7. Chunking" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "JjTVvvoDtN3n", + "colab_type": "code", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 374 + }, + "outputId": "99d48872-13b9-415d-87c8-47b835e20e12" + }, + "source": [ + "from nltk.tokenize import word_tokenize\n", + "\n", + "content = \"Cake is a form of sweet food made from flour, sugar, and other ingredients, that is usually baked.\"\n", + "\n", + "tokenized_text = nltk.word_tokenize(content)\n", + "tagged_token = nltk.pos_tag(tokenized_text)\n", + "\n", + "grammer = \"NP: {
?*}\"\n", + "phrases = nltk.RegexpParser(grammer)\n", + "\n", + "result = phrases.parse(tagged_token)\n", + "print(result)\n", + "\n", + "#result.draw()" + ], + "execution_count": 21, + "outputs": [ + { + "output_type": "stream", + "text": [ + "(S\n", + " Cake/NNP\n", + " is/VBZ\n", + " (NP a/DT form/NN)\n", + " of/IN\n", + " (NP sweet/JJ food/NN)\n", + " made/VBN\n", + " from/IN\n", + " (NP flour/NN)\n", + " ,/,\n", + " (NP sugar/NN)\n", + " ,/,\n", + " and/CC\n", + " other/JJ\n", + " ingredients/NNS\n", + " ,/,\n", + " that/DT\n", + " is/VBZ\n", + " usually/RB\n", + " baked/VBN\n", + " ./.)\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "jhae39lQuZCy", + "colab_type": "text" + }, + "source": [ + "##8. Bag of Words" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "mJ1HKNB1tqL1", + "colab_type": "code", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 131 + }, + "outputId": "9dacc291-4676-4226-99cd-ba23070c42ca" + }, + "source": [ + "from sklearn.feature_extraction.text import CountVectorizer\n", + "import pandas as pd\n", + "\n", + "content = \"\"\"Cake is a form of sweet food made from flour, sugar, and other ingredients, that is usually baked.\n", + "In their oldest forms, cakes were modifications of bread, but cakes now cover a wide range of preparations that can be simple or elaborate, and that share features with other desserts such as pastries, meringues, custards, and pies.\"\"\"\n", + "\n", + "count_vectorizer = CountVectorizer()\n", + "\n", + "bag_of_words = count_vectorizer.fit_transform(content.splitlines())\n", + "\n", + "pd.DataFrame(bag_of_words.toarray(), columns = count_vectorizer.get_feature_names())" + ], + "execution_count": 22, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
andasbakedbebreadbutcakecakescancovercustardsdessertselaboratefeaturesflourfoodformformsfrominingredientsismademeringuesmodificationsnowofoldestorotherpastriespiespreparationsrangesharesimplesuchsugarsweetthattheirusuallywerewidewith
0101000100000001110101210001001000000011101000
1210111021111110001010001112111111111100210111
\n", + "
" + ], + "text/plain": [ + " and as baked be bread but ... that their usually were wide with\n", + "0 1 0 1 0 0 0 ... 1 0 1 0 0 0\n", + "1 2 1 0 1 1 1 ... 2 1 0 1 1 1\n", + "\n", + "[2 rows x 45 columns]" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 22 + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "t-PUyYohulkQ", + "colab_type": "text" + }, + "source": [ + "###8.1 Synonyms using wordnet" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "c9O3VvmLueas", + "colab_type": "code", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 85 + }, + "outputId": "bf680489-b5f6-4c18-8374-a700fc13a31f" + }, + "source": [ + "from nltk.corpus import wordnet\n", + "\n", + "syns = wordnet.synsets(\"dog\") \n", + " \n", + "print(syns[0].name()) \n", + " \n", + "print(syns[0].lemmas()[0].name()) \n", + " \n", + "print(syns[0].definition()) \n", + " \n", + "print(syns[0].examples())" + ], + "execution_count": 23, + "outputs": [ + { + "output_type": "stream", + "text": [ + "dog.n.01\n", + "dog\n", + "a member of the genus Canis (probably descended from the common wolf) that has been domesticated by man since prehistoric times; occurs in many breeds\n", + "['the dog barked all night']\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "nLdgG6MjuyUJ", + "colab_type": "text" + }, + "source": [ + "###8.2 Frequency distribution of words" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "okhy1hfMurbI", + "colab_type": "code", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 337 + }, + "outputId": "3f94e773-f3a2-4963-9af6-262f4a6edf2f" + }, + "source": [ + "import matplotlib.pyplot as plt\n", + "\n", + "content = \"\"\"Cake is a form of sweet food made from flour, sugar, and other ingredients, that is usually baked.\n", + "In their oldest forms, cakes were modifications of bread\"\"\"\n", + "\n", + "words = nltk.tokenize.word_tokenize(content)\n", + "\n", + "fd = nltk.FreqDist(words)\n", + "fd.plot()" + ], + "execution_count": 24, + "outputs": [ + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAFACAYAAACx2ns2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deZhcZZn+8e+dfU8giZBBSBQRRAShgyziAooLOiCICy4gOuI2iuLCODogjDOOM46Oyk9U3ADRURGUxAVxAQQE7CCEsMRhkU0QEiBkJ8vz++M91V1dqVNd1alT1Z1zf66rru469Z5TT3dXn+e861FEYGZm5TWq2wGYmVl3ORGYmZWcE4GZWck5EZiZlZwTgZlZyTkRmJmV3JhuB9CqWbNmxbx584a079q1a5k4ceKILO9Yhn8srZZ3LO0p71ias2jRomURMbvuixExoh49PT0xVL29vSO2vGMZ/rG0Wt6xtKe8Y2kO0Bs551U3DZmZlZwTgZlZyTkRmJmVnBOBmVnJFZ4IJI2W9CdJC+u8Nl7SDyTdIek6SfOKjsfMzAbqRI3gZOC2nNfeATwWEc8AvgB8tgPxmJlZlUITgaSnAq8CvpFT5Cjg3Oz7C4GXSFIRsaxav5G/rtzI42ueLOLwZmYjVtE1gv8BPgZsznl9J+A+gIjYCKwAZhYRyOk/vYX3/3IZl97yUBGHNzMbsRQF3ZhG0quBIyLivZJeDHwkIl5dU2YJ8IqIuD97fidwQEQsqyl3EnASwJw5c3oWLFjQcjznL17JT5au5k17TeG1z5rS1D5r1qxh0qRJTb9HkeUdy/CPpdXyjqU95R1Lc+bPn78oIubXfTFvptnWPoDPAPcDfwEeAtYA360pcylwUPb9GGAZWXLKewx1ZvE5V94Zc09dGKf/dEnT+wynWYKOZfjH0mp5x9Ke8o6lOXRjZnFEfDwinhoR84A3Ar+NiLfUFLsEOCH7/tisTCFVlFlTxgOwfLX7CMzMqnV80TlJZ5Iy0yXAN4HzJd0BPEpKGIWYOWUcAMtWri/qLczMRqSOJIKIuBy4PPv+tKrt64DXdSKGSo1g2SonAjOzaqWZWexEYGZWX2kSwfaTxyHgsTUb2LgpbzSrmVn5lCYRjB4lpo5PP+6j7jA2M+tTmkQAMCNLBI+4ecjMrE+pEsH0CenHXbbKNQIzs4pSJYIZWSJY7hqBmVmfUiWC6eMrNQInAjOzinIlAjcNmZltoVSJoNJZ7NnFZmb9SpUIpk8YDcAyDx81M+tTrkTgGoGZ2RZKlQhmTHBnsZlZrVIlgulVM4s3by7mhjxmZiNNqRLB2NFi2oQxbNwcrFi7odvhmJkNC6VKBOBVSM3MapU2EXi9ITOzpHyJYGq6U9lyTyozMwPKmAjcNGRmNkDpEsHMyU4EZmbVSpcIKk1Dy1a6acjMDMqYCLKmoeWrXSMwM4NSJoJUI3jEncVmZkApE0HWR+D1hszMgDInglXrifAyE2ZmhSUCSRMkXS/pJkm3SDqjTpm3SXpE0o3Z4x+Kiqdi8vgxTBw7mvUbN7P6yU1Fv52Z2bA3psBjrwcOi4hVksYCV0n6RURcW1PuBxHxjwXGsYWZU8Zx/2NrWbZyPVPGF/krMDMb/gqrEUSyKns6NnsMi7YYTyozM+tXaB+BpNGSbgQeBi6LiOvqFHutpMWSLpS0c5HxVDgRmJn1Uyc6TCXNAC4G3h8RS6q2zwRWRcR6Se8C3hARh9XZ/yTgJIA5c+b0LFiwYEhxrFmzhkmTJnF27wp+ffdaTtpvGi/fddKg5Vs9fhHlHcvwj6XV8o6lPeUdS3Pmz5+/KCLm130xIjryAE4DPtLg9dHAisGO09PTE0PV29sbERH/9cvbY+6pC+MLly1tqnyrxy+ivGMZ/rG0Wt6xtKe8Y2kO0Bs559UiRw3NzmoCSJoIHA7cXlNmTtXTI4HbioqnWmVSmZuGzMyKHTU0BzhX0mhSX8QPI2KhpDNJmekS4AOSjgQ2Ao8Cbyswnj6zplYmlXl2sZlZYYkgIhYD+9bZflrV9x8HPl5UDHm83pCZWb/SzSyG6qYh1wjMzEqaCLzekJlZRSkTwfSJYxk7Wqxcv5F1G7zMhJmVWykTgaS+O5UtX+3mITMrt1ImAkjrDYGbh8zMSpsIvMyEmVniROBEYGYlV95EMNVDSM3MoMyJYLJrBGZmUOZE4BqBmRlQ5kTgSWVmZoATgdcbMrPSK20imOn1hszMgBIngu0njUOCx9Y8ycZNm7sdjplZ15Q2EYwZPYrtJ40jAh5d41qBmZVXaRMBVHcYOxGYWXmVOhHM9C0rzczKnQi8zISZmRMBAMs9csjMSqzciWCqm4bMzMqdCLL1hh5xIjCzEit3IvB6Q2ZmJU8EfX0ErhGYWXk5EeA+AjMrt8ISgaQJkq6XdJOkWySdUafMeEk/kHSHpOskzSsqnnq2n5yahpavepLNm6OTb21mNmwUWSNYDxwWEfsAzwVeIenAmjLvAB6LiGcAXwA+W2A8W5gwdjRTJ4xh4+ZgxdoNnXxrM7Nho7BEEMmq7OnY7FF72X0UcG72/YXASySpqJjqme3lqM2s5ArtI5A0WtKNwMPAZRFxXU2RnYD7ACJiI7ACmFlkTLUq/QSPeL0hMyspRRTfNi5pBnAx8P6IWFK1fQnwioi4P3t+J3BARCyr2f8k4CSAOXPm9CxYsGBIcaxZs4ZJkyYN2PZf1zzGtQ+s55QDp/P8nScOWr7V47ervGMZ/rG0Wt6xtKe8Y2nO/PnzF0XE/LovRkRHHsBpwEdqtl0KHJR9PwZYRpac8h49PT0xVL29vVts++TFN8fcUxfGt666q6nyrR6/XeUdy/CPpdXyjqU95R1Lc4DeyDmvFjlqaHZWE0DSROBw4PaaYpcAJ2TfHwv8Ngu4Y7zekJmV3ZgCjz0HOFfSaFJfxA8jYqGkM0mZ6RLgm8D5ku4AHgXeWGA8dXm9ITMru8ISQUQsBvats/20qu/XAa8rKoZmzJzsSWVmVm6lnlkMMDurETzipiEzK6nSJwKvN2RmZedEULXeUIf7qc3MhoXSJ4JJ40YzYewo1m3YzOonN3U7HDOzjit9IpDUXytY6eYhMyuf0icCqOon8HpDZlZCTgTArCnZyCGvN2RmJeREgG9QY2bl5kSAE4GZlZsTAf1NQ15vyMzKyIkAmOkagZmVmBMBbhoys3JrORFI2k7S3kUE0y2z+1YgddOQmZVPU4lA0uWSpknaHrgBOEfS54sNrXNcIzCzMmu2RjA9Ip4AjgHOi4gDgJcWF1ZnTZswljGjxMp1G1m3wctMmFm5NJsIxkiaA7weWFhgPF0xapSYWRk5tNrNQ2ZWLs0mgjNI9xe+IyL+KOnpwP8VF1bneTlqMyurZu9Q9mBE9HUQR8Rd21IfAbifwMzKq9kawZeb3DZiVZqGlnm9ITMrmYY1AkkHAQcDsyWdUvXSNGB0kYF12uysRvCIawRmVjKDNQ2NA6Zk5aZWbX8COLaooLqhv4/ANQIzK5eGiSAirgCukPSdiLinQzF1xay+SWWuEZhZuTTbWTxe0teBedX7RMRhRQTVDTMnu7PYzMqp2UTwI+CrwDeAbXLGlUcNmVlZNZsINkbE2a0cWNLOwHnADkAAX4+IL9aUeTHwU+DubNNFEXFmK+/TLpWmIfcRmFnZNJsIFkh6L3Ax0HfJHBGPNthnI/DhiLhB0lRgkaTLIuLWmnK/j4hXtxR1AbafNA4JHl3zJBs3bWbMaC/Mambl0GwiOCH7+tGqbQE8PW+HiHgQeDD7fqWk24CdgNpEMCyMGT2K7SaN49HVT/Lomid5ytQJ3Q7JzKwjmkoEEfG0rXkTSfOAfYHr6rx8kKSbgL8CH4mIW7bmvbbGrCkpESxb6URgZuWhiBi8kHR8ve0RcV4T+04BrgD+LSIuqnltGrA5IlZJOgL4YkTsVucYJwEnAcyZM6dnwYIFg8Zcz5o1a5g0aVLu65+64lFufvhJTnvhduyzw/hBy7d6/K0p71iGfyytlncs7SnvWJozf/78RRExv+6LETHog7ScROVxDnAXcGET+40lLVZ3SpPv8xdgVqMyPT09MVS9vb0NX3//926IuacujItuuK+p8q0ef2vKO5bhH0ur5R1Le8o7luYAvZFzXm22aej91c8lzQD+t9E+kgR8E7gtIuouUCdpR+BvERGSnkda+2h5MzEVwesNmVkZNdtZXGs1MFi/wfOBtwI3S7ox2/bPwC4AEfFV0jIV75G0EVgLvDHLXF3huQRmVkZNJQJJC0ijhCAtNvcs4IeN9omIqwANUuYs4KxmYuiE2X2JwDUCMyuPZmsEn6v6fiNwT0TcX0A8XeX1hsysjJqaNRVp8bnbSSuQbgdsk5fMXm/IzMqoqUQg6fXA9cDrSPctvk7SNrUMNcCsqU4EZlY+zTYNfQLYPyIeBpA0G/g1cGFRgXXDzMn96w11sc/azKyjml1QZ1QlCWSWt7DviDFh7GimThjDxs3BirUbuh2OmVlHNFsj+KWkS4HvZ8/fAPy8mJC6a9aU8axct9HNQ2ZWGoPds/gZwA4R8VFJxwCHZC/9Abig6OC6YdaUcdy9bDWPrHyScd0OxsysAwZr3vkf0v2JiYiLIuKUiDiFtBz1/xQdXDf03bt4tWsEZlYOgyWCHSLi5tqN2bZ5hUTUZX2zi1c6EZhZOQyWCGY0eG1iOwMZLvrWG/LsYjMricESQa+kd9ZulPQPwKJiQuourzdkZmUz2KihDwIXS3oz/Sf++cA44OgiA+uWWQPWG2q4VJKZ2TahYSKIiL8BB0s6FNgr2/yziPht4ZF1yewB6w35LmVmtu1r9n4EvwN+V3Asw8LA9YacCMxs27fNzQ7eWpX1hpa7s9jMSsKJoMbkcaOZMHYUazdsYu3Gzd0Ox8yscE4ENST1dRivWOdEYGbbPieCOmZmieBxJwIzKwEngjpmZ5PKVqx3IjCzbZ8TQR1uGjKzMnEiqKOyzMTjrhGYWQk4EdThGoGZlYkTQR19iWD9pi5HYmZWPCeCOmZ51JCZlUhhiUDSzpJ+J+lWSbdIOrlOGUn6kqQ7JC2WtF9R8bRiVmXUkBOBmZVAs/csHoqNwIcj4gZJU4FFki6LiFuryrwS2C17HACcnX3tqr4agTuLzawECksEEfEg8GD2/UpJtwE7AdWJ4CjgvIgI4FpJMyTNyfbtmukTxzJmlFizIbj4T/czSoMvRy2JiWvdp2BmI0+RNYI+kuYB+wLX1by0E3Bf1fP7s21dTQSjRokdpk3ggcfX8qEf3NT0fs/cfiyHH1JgYGZmBVC6GC/wDaQpwBXAv0XERTWvLQT+IyKuyp7/Bjg1Inpryp0EnAQwZ86cngULFgwpljVr1jBp0qSmyl73wDqu/MsqRo9uLldefd86RgsuOGYHxo5q7oY2rcTTStmiyzuW9pR3LO0p71iaM3/+/EURMb/uixFR2AMYC1wKnJLz+teA46qeLwXmNDpmT09PDFVvb29h5V/4n7+NuacujNseXFHI8YuM3bF0p7xjaU95x9IcoDdyzqtFjhoS8E3gtoj4fE6xS4Djs9FDBwIrosv9A0O1+w5TAVj60MouR2Jm1poi5xE8H3grcJikG7PHEZLeLendWZmfA3cBdwDnAO8tMJ5C7TFnGgC3PehEYGYjS5Gjhq5ikLu/Z9WV9xUVQyftsWOlRvBElyMxM2uNZxa3SSUR3O6mITMbYZwI2mTuzMmMGw0PrljHijUbuh2OmVnTnAjaZPQosfO01NJ2u5uHzGwEcSJoo7nTxwKw9G9uHjKzkcOJoI3mTk81Ao8cMrORxImgjXbJEoFHDpnZSOJE0EZzZ2RNQw+tZPPmYpfuMDNrFyeCNpo+fhSzpoxn9ZObeODxtd0Ox8ysKU4EbfasOWk+wW0PunnIzEYGJ4I285pDZjbSOBG0WWXNods9hNTMRggngjbrW2rCTUNmNkI4EbTZM54yhVGCu5etZt0G37rSzIY/J4I2mzB2NE+bNZnNAXc8vKrb4ZiZDcqJoAB77Jj1E7jD2MxGACeCArifwMxGEieCAuxeuUmNRw6Z2QjgRFCAZ/m2lWY2gjgRFGCnGROZPG40y1atZ9mq9d0Ox8ysISeCAowapf7mIXcYm9kw50RQkN09csjMRggngoJUFp/zyCEzG+6cCArSt/icRw6Z2TDnRFCQyqSypQ+tZJNvUmNmw1hhiUDStyQ9LGlJzusvlrRC0o3Z47SiYumG6ZPGMmf6BNZv3Mw9y1d3Oxwzs1xF1gi+A7xikDK/j4jnZo8zC4ylK/pmGLvD2MyGscISQURcCTxa1PFHAo8cMrORoNt9BAdJuknSLyQ9u8uxtJ1HDpnZSKCI4joyJc0DFkbEXnVemwZsjohVko4AvhgRu+Uc5yTgJIA5c+b0LFiwYEjxrFmzhkmTJnWs/D0rNnDKr5az4+TR/L8jZm/V8Tsdu2MpvrxjaU95x9Kc+fPnL4qI+XVfjIjCHsA8YEmTZf8CzBqsXE9PTwxVb29vR8uv37ApnvHPP4u5py6MVes2bNXxOx27Yym+vGNpT3nH0hygN3LOq11rGpK0oyRl3z+P1Ey1vFvxFGHcmFHsOnsKAH/2fAIzG6aKHD76feAPwO6S7pf0DknvlvTurMixwBJJNwFfAt6YZa1tyh5ec8jMhrkxRR04Io4b5PWzgLOKev/hIo0c+qtHDpnZsNXtUUPbvD0qI4ce8sghMxuenAgKVj2pbBts+TKzbYATQcF2nDaBaRPG8PiaDTy80jepMbPhx4mgYJLYo+/WlW4eMrPhx4mgAzxyyMyGMyeCDtjDaw6Z2TDmRNABu3sVUjMbxpwIOqCSCO54eCUbNm3ucjRmZgM5EXTAlPFj2Hn7iWzYFNy9zDepMbPhxYmgQyr9BB45ZGbDjRNBh3jkkJkNV04EHeKRQ2Y2XDkRdMjurhGY2TDlRNAh82ZOYvyYUTzw+FpWrN3Q7XDMzPo4EXTImNGj2G0H36TGzIYfJ4IO6usn8MghMxtGnAg6aA/PMDazYciJoIM8csjMhiMngg6qjBz6s29SY2bDiBNBB82eOp5ZU8axcv1GHnh8bbfDMTMDnAg6rm8l0gfdPGRmw4MTQYdV+gmWegipmQ0TTgQdVqkRePE5MxsunAg67FmVGoFHDpnZMFFYIpD0LUkPS1qS87okfUnSHZIWS9qvqFiGk912mMIowV3LVrNhk0cOmVn3FVkj+A7wigavvxLYLXucBJxdYCzDxoSxo5k3azKbNgf3r9zY7XDMzBhT1IEj4kpJ8xoUOQo4L9KA+mslzZA0JyIeLCqm4WKPHady1yOruWTpapaPubOpfe5/YDW9q5orW3R5x9Ke8o6lPeXLFMvq5Wvp6Wm6eNMKSwRN2Am4r+r5/dm2bT4RPPvvpvPzmx/iynvXceW9tze/4+IWyhZd3rG0p7xjaU/5ksSy2/ZjOaW1ozelm4mgaZJOIjUfMWfOHBYtWjSk46xZs6alfYsq/5wJm3nDs6ewcu2TjBnT3J9g48aNTZcturxjaU95x9Ke8mWKZfqYzUM+/zUUEYU9gHnAkpzXvgYcV/V8KTBnsGP29PTEUPX29o7Y8o5l+MfSannH0p7yjqU5QG/knFe7OXz0EuD4bPTQgcCKKEH/gJnZcFNY05Ck7wMvBmZJuh84HRgLEBFfBX4OHAHcAawBTiwqFjMzy1fkqKHjBnk9gPcV9f5mZtYczyw2Mys5JwIzs5JzIjAzKzknAjOzklOMsFsmSnoEuGeIu88Clo3Q8o5l+MfSannH0p7yjqU5cyNidt1X8iYYbIsPGkyoGO7lHcvwj2Ukxz6cYhnJsQ+nWFp5uGnIzKzknAjMzEqubIng6yO4vGPp/LGLLu9Y2lPesWylEddZbGZm7VW2GoGZmdVwIjAzKzknAjOzkitdIpA0R9L4rTzG+dnXk9sTVcP32k7S8yS9sPJo03Gf38y2kU7SKEkHF3Ts0ZI+VMSxs+Nv8Tkd7LMr6e8ljaj/6+xv9PoW9ynF57dTStdZLOnXwK7AjyPiI3Vefz5wY0SslvQWYD/gixFxT1WZW4GXAr8g3XNB1ceIiEdz3vuZwNnADhGxl6S9gSMj4tM55f8BOBl4KnAjcCDwh4g4rE7Z0cCrSHeF61tePCI+n3PsGyJivya2bV9v/6rj5/2sk4APA7tExDsl7QbsHhELc8rvXSf2i+qUGw3cEhF7NIqrZp8/RcS+zZZvhaTrI+J5LZT/T+DTwFrgl8DewIci4rt1yjb1N6p5/bvAQcCPgW9FxBY3xJV0TKMY6/3eG7zfjhHxUIPXD2bLv+t5dcr1RsT8Ft636d9NK7/zrPzJwLeBlcA3gH2Bf4qIX+WUb/Wz3tTxJd0M5J6gI2LvvNdaNSLuWdxOEfFSSQL2zClyNrCPpH1If9xvAOcBL6oq81XgN8DTgeobiIr0h3t6zrHPAT5Kuk0nEbFY0vdIH9J6Tgb2B66NiEMl7QH8e07ZBcA64GZgc04ZJB0EHAzMllR9H+xpwOg6uyzKfiYBuwCPZd/PAO4FnpbzVt/O9j0oe/4A8CNgi38OSd8i/XPeUhV7AFuckCJik6SlknaJiHvzfs4av5H0WuCiaHDlI2kljf/xptXZfLWks4AfAKuryt6Qc5iXRcTHJB0N/AU4BrgS6DspSdoR2AmYKGlf+i80pgGT8uLL3vctkqYBxwHfkRSkv8X3I2JlVuzvs69PIX0Wfps9PxS4hjq/9wa+SboA2UJWc96VdBGzqRIi6f+p1q8lfYQtf48DLjSG8PmFJn7nNd4eEV+U9HJgO+CtwPlA3URAC5/1Fo//6uxr5b4t52df35xz3CErXSKAvpvi3JLz8saICElHAWdFxDclvaNm/y8BX5J0NikpVJprroyImxq89aSIuD7lof73a1B+XUSsk4Sk8RFxu6Tdc8o+tckrhHHAFNLffmrV9ieAY2sLR8TTACSdA1wcET/Pnr8SeE2D99k1It4g6bjsOGtU84NXOTAi8hJzPdsBt0i6noEnjSNzyr8LOAXYKGkdWcKuPbFHxFQASf8KPEj6xxPpH29OzrGfm309s/pQwBa1tszY7OurgB9FxIo6v5aXA28j1QSra3QrgX/OOW71z/GEpAuBicAHgaOBj0r6UkR8OSJOBJD0K2DPyG4RK2kO8J3Bjl/zXnWTQGZ+dvxmmh3ekH2tvllVvYuqlj6/mcp5rtHvvFrlxSOA8yPilgafXWjts9708SutEJIOr6nR/pOkG4B/avRDtKKUiWAQKyV9HHgL8MKsvXVsTtnbSVcVF5H+uOdLOicivpxTfpmkXcmuOiUdSzrh5Llf0gzgJ8Blkh4jf8G9X0h6WV71tSIirgCukPSdiLhH0qSIWNNon8yBEfHOquP8Iqty53lS0kT6f9ZdgfU5Zf8gac+IuLWJOAD+pclyQP8JvgVHRsQ+Vc/PlnQTcFqdYx/a4rEXSLqd1EzxHkmzSTW56mOeC5wr6bUR8eNWDi7pSNJtX59BuvJ+XkQ8nDVf3ApUfzZ3joH3Cf8bqdbXLkuAHWn8GQf6LziaKDfg8wupjwGYEhFP5Oy2cLDfeY1FWZJ8GvBxSVNpUMumtc/6UI4vSc+PiKuzJwfT7v7dIhYwGskP0gf3FOAF2fNdgONzyi4GJlc9nwwsbnDspwO/Jt2j+QHgKtKKgM3E9SLgSGBczutHk66O15KujlYCTzQ43kGkE8O92fN9gK80KH8p8ElSe+884BPApQ3KHw5cATwCXECqkr+4wc+2Alia/U5vbvR7HOLfdTvgeaTa2wuBFzYoew2pFjCa9A/3ZuCanLI7kJpHfpE93xN4R4Njjwe2B0ZXfWZ2aFD+VcDHSEnoNOC0QX7Oc/N+NuAlNc/Pyv6ub8sevwC+3Mbf+e9ITYmXApdUHjVlDsu+HlPv0eDY3yM1B03OPsf3Ax9tUL76dz4J2LFB2VGkvsEZ2fOZwN7t+KwP8fg9wE3Zce8hNbXt187/j9J1FrdT1pmzf0Ssy55PAP4YEc/JKd8TEYskTQZGRcRKSa+OnE6lFmO5GzgKuDma+KNKuo5Ulb4ksmqnpCURsVdO+e2B06lqBgPOiJzO4myfmaQObpH6OeounyvpDlLyHdC/EVUd9DXlq9vyx5FqbKujfht+S53uWfl5wBeB52fvczXwwYj4S52yvyC1EX8iIvaRNAb4U4PPQCudnF8lnbQOJfVVHQtcHxHvqC07VFm7eXXT5sVtPPaL6m2PdFVfKXNGRJwu6dv1i8bbc459Y0Q8V9KbSSfVfwIWRZ3mUbU+kKLSHPj0iDhT0i6kxHF9nbKjSH+X39DEZ71qv52AuTXxXDnIPtOzcisalRsKNw1lJF0VEYfU6TCs256c+TZwnaTKP89rSFeHec6RdHxELMne843Ah8jvVGrFfcCSZpJARUTcV9M0ualB2UeBk7NqbETEqnrlJNWe0CrNArtkHbz1OlEfiYhLWoi7r6kn+6c9ivRPmKeVTneyE/5RTYYzKyJ+mDUnEhEbJW3xexxiB/DBEbG3pMURcYak/yZdtW8h73NL488vwA3Ayoj4taRJkqZGf6fyVqk+4Tcoc3r29cQWDz9W0ljS/9xZEbFBqWO8nqYGUlT5SlbuMFLfz0rSKKz968S/WdLHIuKHwM+aCVzSZ0l9IrcysBM9NxFIehXwbGBC5X82Is7MK98qJ4JMRBySfW26PTkiPi/pcuCQbNOJEfGnBrscC1wo6U3AC4DjgZcNLeIt3AVcnl2h9rVP5l31APdlbY2R/UOdDNyWd3BJzyG1OW+fPV8GnFBJalX+O/s6gdRZeBPpZLQ30Ev/yIpqf1IaPbWgJvZBR69kie8nkk4nv/OslU53sjbkd7LlFWS9q9PVWc2n0j58IKmZq9ZQOoDXZl/XSPo7YDk5ndatfG4rJL0TOIn0N92VlKi+Cryk1WPVHDdv9FVuUpK0Ayk5/11EvFLSnsBBEZF3YfU1UlPJTcCVkuaSmkTraXYgRcUBEbGfpD+RAn5M0rgG5Zsa8VTlNaThpY36Efrk1Qyb2bdZTgRbKbvCzRsqWFv2rqwW8BPS0MuXRcTaQXZr1t3ZY1z2GMy7Sc0fO5H6K37FwBEbtZIIC3wAABS3SURBVL4GnBIRvwOQ9GLSSogDJmtF1nkq6SJSO+bN2fO9gE/lHHsiKQFUJ8W6w0ezY1WPgx9FSjiNOv9a6XQH+Cnwe1J/Tm4tKXMKqe17V0lXA7OpP/pqKB3AC7O4/4v0GQvSiWALGtp8j/eR+k2uy8r8n6SnNBlbo/dqOSmRRit9m9T3BPBn0om1biKIbORe1aZ7JOV13Dc1kKLKhqw5qZLcZ9O4JtHsiKeKu0jNmU0lAlqoGQ6V+wg6QFtODHkK6apxPWz9xJDsQ3teRLR9fHHVe9wUA0fS1N1W9dotEfHswbYNMZbq9uSNpCvDcyLi4Sb2fREwHfhlRDyZU+bGiHhuvddyyo8Bdidd8S6NiA0Nyo4HXsuWtY2G1fxsvwl57cNZH1GlKahWRMQWJyVJ10XEAcom3GU/xw1b+3kcCkl/jIj9VTX5r9HfoZUaRNYP8l3SRcMGBmkuy/od3kDqeziXlNg/GRE/2uofNB3/x6TBGb9hYA34AznlK3+na0md6MtJkyqf0Y54wDWCTnn14EWGLtIkq7mSxuWd3GqpxVnOwF2S/oX+SS1vIV3Z5Fks6Rv0T9p5M2lEUL1Ynkoa1lhZIuD3wMkRcX+98q22J9dcLd9cOUyDXRZKOiKyORNNeB79J/b9JBF1Zs9mfkq6CFhEE1eEqpmZm3fsaHL4ZY0rJP0zqd/icOC9pOa5bmi2ia3iOzRfg/g8qUmyqYEUwIWkv89LSEnjNaShtXUpDc09hTSz+CQNMrOY/hFUzWq6ZjhUrhF0QVb9nlB5Hs3PkG10zPOAZ5E+YNXtlHkjI64gm+UczY0a2g44g/7+kN8Dn4qIx3LKTwDew8BRRmdHNsKqpuxlpOGA1UnmzRFxeM6xW0ockv4C7MzAWdEPkf653xkRi2rKryQNS1zPIFeQypk92+DqLvd3vLXHrtpvO2A3Bn7GtuiIVBrx8g5Sk5xIw4HPaSa2dlMaZPBlYC/S/IPZwLERkXfx0HQNQtKVpOGczXQUI+lnwGsqNTuliXYLI6Inp/wPSInj+OyiahJpuHFurTLrc3hm9rRhLbJmv4Y1w6FyjaCDlCb7/Dfwd8DDpOFjt5FGA2ytO7PHKAbOuMzT0izn7IT/AQ0yaqiq/DrgC9ljMLMjorq55zuSPtig/LdJieN12fO3ZNvqJg7gMuDCiLgUQNLLSM0z3yaNEDmgJvapWS1iwMk0RyuzZwGukfScSt9Jm4+dO1SW+jOdPxURp5GWPkFpEb0LimxizBMRN2TNdk01sdFaDaLVgRQ/AX6oNOFzZ9LF1RbrklVpaWZx1r92LqlJU8DOkk6ol6yz8rVrGe0i6QUNahwtcyLorH8l/WP+OmuTPZR0EttqEXEGgKQp2fOGJ2panOWs5kcNVcrvBnyGNMGq+sq0XgfacqUF/r6fPT+O1A6ap9XEUTsr+leSPhcR71L9FT7rnUyvof5omqZnz2YOAd6Wtemvp7+2Ua9dvtVjQ2tDZXeW9PGI+Ex2hfpD0s/bLa00sTXVSZ9paSBFRJyT/T5+ksXzroi4psEurc4s/m/SQJGlWflnkj77dWsctL6WUcucCDprQ0QsV1p2d1RE/E7S/7TjwEqjcs5n4In6+IjIW1PpfaRRP3tIeoD0j9LoSrCpUUNVvk2agPYF0rC3E8mfFv92UrPAF0j/TNdk5fO0mjgelHQq8L/Z8zcAf8s62es1Fwx6MpW0IIt1KnCr0rpH1VebeesevbJBnFt7bGhtqOzbgQuU5kAcSpod3UwNru3ymsGov0Bd0zWI7G/8zGZqORq4iF1lkcUbgQMlHdigBnE6aVXTnSVdQGqyfFuDtxpbSQLZz/JnpSHceVpdy6hlTgSd9Xh2xX4l6R/wYara87fS19nyRH0OOSfqiLgLeKmqZjkPcvzJlWNn+1+e7ZtnYkT8RpIizRD+lKRF1KzXk/2j/vsgJ7darSaON5H+WX+SPb862zYaqLcOfjMn08+1EG+fSOs7HQLsFhHfVhqaOKXOsQV8loEL+1W2NTLoUFkNnPT3RVKSv5rUebxf5K+cWqSmmsGUv4T2M7MaxIAhxy0OpKhtUr0oZ3utE0iTyS4kNUOdHI1nFvdqy4EUvQ3Kt1rjaJk7iztA0jNIa9LcSJokVFm/Zi7ws9rOyiG+R6vDO+8EriV1tP6+Qc2hUv5i0oiF6g7dnog4Oqf8NaRmkAtJyxw/APxHRGxxdSrpKtKaM02NeCpa9rOeSFq58zBSJ/PYiDiiTtnPRsSpg22reu100klv94h4ptJEsR9FRL0brdRbjmJxTjNSvfeqO1RW0u/y9yIiZ+mNIkn6EfCBGLgIXr1ylSbBuktoR8QWI/TU4kCKVmVNvC/IHrsCfyIt1/HFnPLjSTXy6oEXX4mcCWZKI7o+SWpm/RVZjSMiLm9H/OBE0BGSFgIfr+0gzNrd/z0i/r7+ni29R6sn6vGkTtIXkD5Yu5MWeju6ptz5EfHWrNo8j/4Pb2WtobxRQ/uTOsJnkPpGpgH/GRHX1Snb1D+qpC/RQN5omuzEt8UHvZkTXt7JtOr1lk7Wkm4k3YjkhqrRLgPKS3oPaSjn00kDACqmAldHRG6/ktK6OFuINoxMK0JNM9hzSTNmB20GU1q984SoWUI7Il5ep+zp9Y5R6VerU342aaG/ZzOwfyv385LVbPcnJaR3A2ujhZsnNTjukNYyapWbhjpjh3qjRCLiZqUFzoascqImXVXMo786eyWpCSXPJtLQyE2kdvKHs0etnuyq9QTSh7yyhg3Z93mClJTm0r+M9zmkpSZqYz+S1Mwz2IinY0jjxrcjXaU3q3rExwTSiKFG94HoEznr5VSdrHeVVD3EcSqpqSrPkxERytbFyWle+x5p5uhnGLhsxsposMhf5mf0TyybADyNtKpr3ZFpqlrDprIt2riGTROG2gzW9BLaQxhIcQFpTsKrSSf1E0gri9Yl6Tek4cZ/IP0f7h91JjdK+mFEvF45dx6rd/EQQ1jLaCicCDpjRoPXJm7lsYd6on6CNLnq86RZuXmdrdV3Y6tuxxzsbmwXkOYpNFroqxL7vQxcJ79RzJeRc4vQPHWa3q7OOmC3RvXJ+j/ony9xVTReb+qHkr4GzFBa6+ftZMM3q+JdQRoKeVyrQUXNqqdZf8B765VVB9awGUwl0UoaW5t0s3bxPL+RdCn9AwbeQFoSZAtDGEgxM9INqU6O/vsf/LFBLItJI372Iv3dHpf0h9hy+ZjKPc5bnWDa6lpGrYs2rmntR+564t8nTVyq3f4PwA+28tgfIDXBrCd1VFUedwN3NdjvKNJMxStIJ9czqFmvvqb82S3GdVULsa9rJvat+Fm3r3rMAl5BGmXSjr/tyaRkdwZppcrFwPsH2efw7Hf/OeDwDnz+bs7Zvrjm6xRSf1En/zfek/3+Vme/u8rjbuC7g+x7DP1zVY5uUO4a4NCq5y8m5/4S2evXZl8vJS1fvS9wZxM/y1Tg/aTO+fUNyn22mW1Vr91d83m/q9HnfSgP9xF0gNK6KBcDT9J/j+P5pDHNR0eDm3+38B5nR8R7hrDfHqQhjR8EnhIRW1tDqRz3JaQr2tr1VOrdkL6l2IdQvnodng2kiTxnRsRVzR6jwbEXk9a4WZ09n0y610HH1+vJ3r96COQo0pXq9lG/7bzwNWwGo7TG/nYMrRms2fdodSDFq0lNPDuTaqrTSJPv6i6/IekfSX1tPaTPVmUAxm9zyrfarzSRVKs7hPQ5/j3w1WjfgpVuGuqEiPgbcHA2uqCyvMDP8j4oQ3yPlpKA+he+upPUn/BW2tsscCKwB6l/YLAb0rcU+xAS3qmkzt4nlNZL2o90l7h2EANXKN1EnSYrDe1+F0NR3ceykTTpKG+108oaNv9J/wVKW9ewGUy02AxW9furbgKFxr/HVtfJeh2pRrsEOFRplvnnyF+HaQKpiXVRROT2PVUPAqjTr3R1g3jOJTWLVgZLvCnbVm/o85C4RlBSkj5GuqqonBz3Bf41Grdvt3L8pVFnqGg3VK62svH7/0r6pz4tIg4YZNdmjn0KqX+m+uZE34mItkwU3Boa5F6+2ZXme0hXs5UrzbrrQQ1Hkp5Lih3ScM2bal4f6oi3vvWLGm0bQrxDqv1IujUi9hxs29Zo7w2QbSR5S5YEDiGNlf8mqWO4Xa5RWhp4OKhcsb+K1DH+M5q7Z8OgIg1xPRF4NHucWC8JSNq+0aMdsWTv8z1J07ImqiWkmckfzSl+LmnE0JdITSB7kjOTd7iR9AHSFf4s0vIS50t6f02x6oEUnyL1zbyU1JfTaKDBKKWF+yrvtT1taD2JiBUR8ZeIOC7SJMu1pAQ8JW/Yb+YGpbWUKvEcQOMJaC1zjaCk1L8G/WdInYnfa8dVT9XxbyNNrmlmTZ1CKc3jeIB0ItiP9A94fV4bcUExVPdT7MLAlVDvjaEtI13vfVq5l2/hV5pFaaZvJksW7yGNbHugendy7tGQ7Xc86a5xlfsPvA74t4g4v175IcT+96SmpAGLT8aW9++oDDMdS5rnc2/2fC5wezv/Tu4jKK8HsmGMhwOfVZpg1s4a4ivaeKyt9XpSPJ+LiMezyUd5V8mFqJzoJZ0DXBzZvQ4kvZKB4+e3Vr17+eaVvUFpDZ1rs1jafqVZoEH7ZiK7i1mrgwsi4jxJvfSv2HpMRNy6tQFX+TTNLT5Z6H1MqrlGUFJKS9u+glQb+L/s5PicaP52fjYEkm6OLcf6b7FtK47/flLn+GJSU9gupGGYL6gq07ErzaIM576ZwUjqjYj5km4C9o00aSx3FFNHYnIiMOucbBLU7xm44NgL6w3vHOLxq5dTCFItb3RE/EtVmbmNjpG1Xw972WS5vvV62jXQoWiSfk1KXJ8h9XE8TJqNnLeSb/ExORGYdU7W8Xg6aSZykEawnNnGMfMfrno6gdS8cFtENFpuxDoo68+oXnxyOnBB5M/uLz4mJwKzzpM0udLRWfD7jCfdgvLFRb+XNUfS04AHK8N0s2G8O0TEX7oVk4ePmnWQpIMl3UpaKgNJ+0j6SoFvOYl0pzUbPn7EwPW3NtE/QqkrPGrIrLO+ALyctOQ2EXGTpBc23qV5Griy5WjSGPtOriZqgxsTVUuaR8STSrfG7F5A3XxzszKKiPtqhnRuyis7BNVDDjcCf2u07IF1xSOSjoyISwAkHQW09f4CrXIiMOus+yQdDEQ23v9ksmaidhgpI35K7t2kW9WeRZr7cB9wfDcDcmexWQdJmkW6T/BLSSeBX5Hucdu1ESPWHWr+RjnFx+JEYNYZSrczPC8i3tztWKzzJL0lIr5bs1R4n2jTPZSHwk1DZh0SEZskzZU0Lurc/9i2eZOyr41ux9oVTgRmnXUX6VaZlzDwtoNduxq0jtk1+3prRHR1uGgtzyMw66w7STeLGUW6Mqw8bNt3hNJwsY93O5Ba7iMwM+sASf8FvJN0b+jqO+S1+y51LXMiMOsgSQsYeItFSLdq7AW+NlLuDmZDJ+mnEXFUt+Oo5kRg1kGSvkia7fv9bNMbSPejDWBaRLy1W7FZeTkRmHWQpD9GxP71tkm6pfYuVbbtkHRVRBwiaSX9d6vr+9rNpiGPGjLrrCmSdomIewGye9VOyV7zkNJtWEQckn0ddoMDnAjMOuvDwFWS7iRdCT4NeG+2Rv25XY3MCpXdiyJXu+5JMRRuGjLrsOweAXtkT5e6g7gcJN1Nf1PQLsBj2fczgHsr97XuBtcIzDqvB5hH+v/bRxIRcV53Q7KiVU70ks4BLo6In2fPX0m6dWXXuEZg1kGSzifNML2R/uWnIyI+0L2orJMk3RwRzxlsWye5RmDWWfOBPcNXYGX2V0mfBL6bPX8z8NcuxuMlJsw6bAmwY7eDsK46jjSX5OLs8ZRsW9e4acisgyT9DngucD2wvrI9Io7sWlDWFZKmkpoFu34/AjcNmXXWp7odgHWXpOcA5wHbZ8+XASdExJKuxeQagZlZ50i6BvhERPwue/5i4N8j4uBuxeQ+ArMOkHRV9nWlpCeqHislPdHt+KyjJleSAEBEXA5M7l44rhGYmXWUpIuBG4Dzs01vAXoi4uhuxeQagZlZZ72dNGrox9ljFnBiNwNyIjAz66xdgZ1J599xwEuAK7sZkJuGzMw6SNJS4COkOSWbK9sj4p5uxeTho2ZmnfVIRCzodhDVXCMwM+sgSS8hzST+DQMnFV7UrZhcIzAz66wTScuQj6W/aSiAriUC1wjMzDpI0tKI2L3bcVTzqCEzs866RtKe3Q6immsEZmYdJOk20hDSu0l9BJWb1+/dtZicCMzMOkfS3Hrbuzl81InAzKzk3EdgZlZyTgRmZiXnRGClJukTkm6RtFjSjZIOKPC9Lpc0v6jjmw2VJ5RZaUk6CHg1sF9ErJc0i7QImFmpuEZgZTYHWBYR6wEiYllE/FXSaZL+KGmJpK9LEvRd0X9BUq+k2yTtL+kiSf8n6dNZmXmSbpd0QVbmQkmTat9Y0ssk/UHSDZJ+JGlKtv0/JN2a1VA+18HfhZWYE4GV2a+AnSX9WdJXJL0o235WROwfEXsBE0m1hoonI2I+8FXgp8D7gL2At0mamZXZHfhKRDwLeAJ4b/WbZjWPTwIvjYj9gF7glGz/o4FnZ2PKP13Az2y2BScCK62IWAX0ACcBjwA/kPQ24FBJ10m6GTgMeHbVbpdkX28GbomIB7MaxV2kNeYB7ouIq7PvvwscUvPWBwJ7AldLuhE4AZgLrADWAd+UdAywpm0/rFkD7iOwUouITcDlwOXZif9dwN7A/Ii4T9KngAlVu1RWi9xc9X3leeX/qXZyTu1zAZdFxHG18Uh6HulGJccC/0hKRGaFco3ASkvS7pJ2q9r0XGBp9v2yrN3+2CEcepesIxrgTcBVNa9fCzxf0jOyOCZLemb2ftMj4ufAh4B9hvDeZi1zjcDKbArwZUkzgI3AHaRmosdJd496CPjjEI67FHifpG8BtwJnV78YEY9kTVDflzQ+2/xJYCXwU0kTSLWGU4bw3mYt8xITZm0kaR6wMOtoNhsR3DRkZlZyrhGYmZWcawRmZiXnRGBmVnJOBGZmJedEYGZWck4EZmYl50RgZlZy/x+OwOWQ44KODwAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "tags": [], + "needs_background": "light" + } + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "N3BmcD2Uu85e", + "colab_type": "text" + }, + "source": [ + "###9. Word Embeddings" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "kbEUVpk3u30F", + "colab_type": "code", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 564 + }, + "outputId": "a4ad4e2e-c6d9-419e-ec4d-6af060008841" + }, + "source": [ + "from gensim.models import Word2Vec\n", + "\n", + "# define training data\n", + "content=\"\"\"Cake is a form of sweet food made from flour, sugar, and other ingredients, that is usually baked.\n", + "In their oldest forms, cakes were modifications of bread, but cakes now cover a wide range of preparations that can be simple or elaborate, and that share features with other desserts such as pastries, meringues, custards, and pies.\"\"\"\n", + "sentences=nltk.sent_tokenize(content)\n", + "words=[]\n", + "\n", + "for i in sentences:\n", + " words.append(nltk.word_tokenize(i))\n", + "\n", + "# train model\n", + "model = Word2Vec(words, min_count=1)\n", + "\n", + "# summarize the loaded model\n", + "print(model)\n", + "\n", + "# summarize vocabulary\n", + "word_vec_words = list(model.wv.vocab)\n", + "print(word_vec_words)\n", + "\n", + "# access vector for one word\n", + "print(model['sugar'])\n", + "\n", + "# save model\n", + "model.save('model.bin')\n", + "\n", + "# load model\n", + "new_model = Word2Vec.load('model.bin')\n", + "print(new_model)" + ], + "execution_count": 25, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Word2Vec(vocab=48, size=100, alpha=0.025)\n", + "['Cake', 'is', 'a', 'form', 'of', 'sweet', 'food', 'made', 'from', 'flour', ',', 'sugar', 'and', 'other', 'ingredients', 'that', 'usually', 'baked', '.', 'In', 'their', 'oldest', 'forms', 'cakes', 'were', 'modifications', 'bread', 'but', 'now', 'cover', 'wide', 'range', 'preparations', 'can', 'be', 'simple', 'or', 'elaborate', 'share', 'features', 'with', 'desserts', 'such', 'as', 'pastries', 'meringues', 'custards', 'pies']\n", + "[ 1.2900397e-03 -4.4050827e-03 -3.3263196e-03 -2.0251162e-03\n", + " -1.9256959e-03 4.5178290e-03 4.6570050e-03 2.8711003e-03\n", + " -4.2531281e-03 -3.7845614e-04 3.8574554e-03 -1.9891351e-03\n", + " 4.5824298e-03 1.5097363e-03 3.1842457e-03 3.1077143e-04\n", + " -4.2730705e-03 1.6763955e-03 -5.4607273e-04 -4.6840194e-04\n", + " -1.2934895e-03 4.8788548e-03 3.9951564e-03 1.8809187e-03\n", + " -4.5051551e-04 -3.6797088e-03 -2.6806856e-03 -4.9093249e-03\n", + " 1.0661986e-05 -2.5036826e-03 3.6932496e-04 -2.4800377e-03\n", + " -4.7568125e-03 -4.7006006e-03 4.6129087e-03 6.1959872e-04\n", + " 3.6717989e-03 -2.4429176e-03 -1.2363732e-04 -3.5046753e-03\n", + " -2.0565446e-03 -2.3353908e-03 -2.7658308e-03 -1.8586651e-03\n", + " -3.7130108e-03 -6.0016935e-04 -2.9897159e-03 -3.7880535e-03\n", + " -3.9280793e-03 3.9332923e-03 1.6729704e-03 -4.9394034e-03\n", + " 4.6572643e-03 -4.3637827e-03 -9.3136783e-05 -2.9492262e-03\n", + " -1.9744544e-03 -3.6059276e-03 -3.9423611e-03 -2.9691169e-03\n", + " -1.7650127e-04 -3.2259724e-03 9.9363422e-04 1.0665000e-03\n", + " -2.2204674e-03 3.3678003e-03 4.4750911e-03 2.2151966e-03\n", + " 5.6144886e-04 4.4316403e-03 -4.0480960e-03 2.9111751e-03\n", + " -4.8157568e-03 -3.9936928e-03 1.0673276e-04 -4.9511571e-03\n", + " 3.2988950e-03 3.1355142e-03 3.5804932e-03 -2.7380732e-03\n", + " -3.0919246e-03 -6.9842924e-04 3.2996046e-03 1.4009699e-04\n", + " -1.5377058e-03 1.6681730e-03 -2.1100568e-03 4.0061162e-03\n", + " 7.1020133e-04 -4.9544768e-05 -4.6256362e-03 -1.6403218e-03\n", + " 4.4054287e-03 -4.3784659e-03 2.6041116e-03 -3.6177207e-03\n", + " 2.0688606e-04 -1.4998632e-03 9.1215217e-04 -3.8086215e-03]\n", + "Word2Vec(vocab=48, size=100, alpha=0.025)\n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:23: DeprecationWarning: Call to deprecated `__getitem__` (Method will be removed in 4.0.0, use self.wv.__getitem__() instead).\n", + "/usr/local/lib/python3.6/dist-packages/smart_open/smart_open_lib.py:254: UserWarning: This function is deprecated, use smart_open.open instead. See the migration notes for details: https://github.com/RaRe-Technologies/smart_open/blob/master/README.rst#migrating-to-the-new-open-function\n", + " 'See the migration notes for details: %s' % _MIGRATION_NOTES_URL\n" + ], + "name": "stderr" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "GE_C9QAZvbHt", + "colab_type": "text" + }, + "source": [ + "##10. Complete example" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "LnILqHx1vCqQ", + "colab_type": "code", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 459 + }, + "outputId": "d921c903-56dd-4655-98d3-cc12a5d85f96" + }, + "source": [ + "import numpy as np\n", + "from keras.utils import to_categorical\n", + "from keras import models\n", + "from keras import layers\n", + "from keras.datasets import imdb\n", + " \n", + "(train_data, train_target), (test_data, test_target) = imdb.load_data(num_words=10000)\n", + "\n", + "dt = np.concatenate((train_data, test_data), axis=0)\n", + "tar = np.concatenate((train_target, test_target), axis=0)\n", + " \n", + "def convert(sequences, dimension = 10000):\n", + " results = np.zeros((len(sequences), dimension))\n", + " for i, sequence in enumerate(sequences):\n", + " results[i, sequence] = 1\n", + " return results\n", + " \n", + "dt = convert(dt)\n", + "tar = np.array(tar).astype(\"float32\")\n", + "\n", + "test_x = dt[:9000]\n", + "test_y = tar[:9000]\n", + "train_x = dt[9000:]\n", + "train_y = tar[9000:]\n", + "\n", + "model = models.Sequential()\n", + "# Input - Layer\n", + "model.add(layers.Dense(50, activation = \"relu\", input_shape=(10000, )))\n", + "# Hidden - Layers\n", + "model.add(layers.Dropout(0.4, noise_shape=None, seed=None))\n", + "model.add(layers.Dense(50, activation = \"relu\"))\n", + "model.add(layers.Dropout(0.3, noise_shape=None, seed=None))\n", + "model.add(layers.Dense(50, activation = \"relu\"))\n", + "# Output- Layer\n", + "model.add(layers.Dense(1, activation = \"sigmoid\"))\n", + "\n", + "model.summary()\n", + "\n", + "# compiling the model\n", + "model.compile(\n", + " optimizer = \"adam\",\n", + " loss = \"binary_crossentropy\",\n", + " metrics = [\"accuracy\"]\n", + ")\n", + "\n", + "results = model.fit(\n", + " train_x,\n", + " train_y,\n", + " epochs= 2,\n", + " batch_size = 500,\n", + " validation_data = (test_x, test_y)\n", + ")\n", + "\n", + "print(\"Test-Accuracy:\", np.mean(results.history[\"val_accuracy\"]))" + ], + "execution_count": 26, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Model: \"sequential_2\"\n", + "_________________________________________________________________\n", + "Layer (type) Output Shape Param # \n", + "=================================================================\n", + "dense_5 (Dense) (None, 50) 500050 \n", + "_________________________________________________________________\n", + "dropout_3 (Dropout) (None, 50) 0 \n", + "_________________________________________________________________\n", + "dense_6 (Dense) (None, 50) 2550 \n", + "_________________________________________________________________\n", + "dropout_4 (Dropout) (None, 50) 0 \n", + "_________________________________________________________________\n", + "dense_7 (Dense) (None, 50) 2550 \n", + "_________________________________________________________________\n", + "dense_8 (Dense) (None, 1) 51 \n", + "=================================================================\n", + "Total params: 505,201\n", + "Trainable params: 505,201\n", + "Non-trainable params: 0\n", + "_________________________________________________________________\n", + "Train on 41000 samples, validate on 9000 samples\n", + "Epoch 1/2\n", + "41000/41000 [==============================] - 5s 126us/step - loss: 0.4352 - accuracy: 0.7917 - val_loss: 0.2614 - val_accuracy: 0.8930\n", + "Epoch 2/2\n", + "41000/41000 [==============================] - 5s 122us/step - loss: 0.2279 - accuracy: 0.9121 - val_loss: 0.2599 - val_accuracy: 0.8968\n", + "Test-Accuracy: 0.8948888778686523\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "KErz2zNGvhNV", + "colab_type": "code", + "colab": {} + }, + "source": [ + "" + ], + "execution_count": 26, + "outputs": [] + } + ] +} \ No newline at end of file