mercari.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "mercari.ipynb",
      "provenance": [],
      "toc_visible": true,
      "authorship_tag": "ABX9TyPAlIOUkU1URcXGvRAIh9uL"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "source": [
        "!kaggle competitions download -c mercari-price-suggestion-challenge"
      ],
      "metadata": {
        "id": "4O4qpG1XFcPM"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!apt-get install p7zip-full\n"
      ],
      "metadata": {
        "id": "MPWkxbvqGUiB"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!p7zip -d train.tsv.7z"
      ],
      "metadata": {
        "id": "TkfYZM2FIbEH"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!p7zip -d test.tsv.7z"
      ],
      "metadata": {
        "id": "GJ5Ssi0mJYH4"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "\n",
        "train = pd.read_csv('./train.tsv', sep='\\t')\n",
        "test = pd.read_csv('./test.tsv', sep='\\t')"
      ],
      "metadata": {
        "id": "rlOVd0wxIjwC"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 1. EDA"
      ],
      "metadata": {
        "id": "sN-34R9NN-uj"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "print(train.shape)\n",
        "print(test.shape)"
      ],
      "metadata": {
        "id": "M-0zOQCxJkSv"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "train.dtypes"
      ],
      "metadata": {
        "id": "hnOQzjiVJ46z"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "train.head()"
      ],
      "metadata": {
        "id": "TFGNwnBQJ7Iq"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Target variable : Price"
      ],
      "metadata": {
        "id": "c9qot6wZKMIB"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "train.price.describe()"
      ],
      "metadata": {
        "id": "kX1C9uXNJ_A4"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns\n",
        "\n",
        "target_df = train['price']\n",
        "plt.figure(figsize=(6, 4))\n",
        "sns.distplot(target_df, kde=False)\n",
        "plt.show()"
      ],
      "metadata": {
        "id": "i_KHP9FTKQWX"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "target_df = train['price']\n",
        "target_df = np.log1p(target_df)\n",
        "sns.distplot(target_df, kde=False)\n",
        "plt.show()"
      ],
      "metadata": {
        "id": "ROPLXVX3M17f"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### 배송비"
      ],
      "metadata": {
        "id": "MZ-WSPf7N58h"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "train['shipping'].value_counts()"
      ],
      "metadata": {
        "id": "1VKEJflnNCYx"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### 카테고리"
      ],
      "metadata": {
        "id": "BZE9pLDHOa0Q"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "print(train['category_name'].nunique(), 'unique categories')"
      ],
      "metadata": {
        "id": "sb1GEOqNOEXv"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "train['category_name'].value_counts()[:5]\n"
      ],
      "metadata": {
        "id": "zxhm4Dx7OuiE"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Split the categories into three different columns."
      ],
      "metadata": {
        "id": "hKYxoo2SPNZ4"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def split_cat(cat):\n",
        "  try:\n",
        "    return cat.split(\"/\")\n",
        "  except:\n",
        "    return (\"No label\", \"No label\", \"No label\")"
      ],
      "metadata": {
        "id": "X0ht6dYePEeX"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "train['general_cat'], train['subcat_1'], train['subcat_2'] = zip(*train['category_name'].apply(lambda x : split_cat(x)))\n",
        "\n",
        "train.head()"
      ],
      "metadata": {
        "id": "FNt9Z89nP6SJ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(\"General category : %d\" % train['general_cat'].nunique())\n",
        "print(\"Sub-category 1 : %d\" % train['subcat_1'].nunique())\n",
        "print(\"Sub-category 2 : %d\" % train['subcat_2'].nunique())"
      ],
      "metadata": {
        "id": "DV8Mwz_oQNUs"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "x = train['general_cat'].value_counts().index.values.astype('str')\n",
        "y = train['general_cat'].value_counts().values\n",
        "pct = [(\"%.2f\"%(v*100))+\"%\"for v in (y/len(train))]"
      ],
      "metadata": {
        "id": "HUilRPzUREE6"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install chart-studio\n",
        "import chart_studio.plotly as py\n",
        "import plotly.graph_objects as go"
      ],
      "metadata": {
        "id": "rKjX38GuUgI2"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import plotly.express as px"
      ],
      "metadata": {
        "id": "3j06RC6bW_fQ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "trace1 = go.Bar(x=x, y=y, text=pct)\n",
        "layout = dict(title= 'Number of Items by Main Category',\n",
        "              yaxis = dict(title='Count'),\n",
        "              xaxis = dict(title='Category'))\n",
        "fig=dict(data=[trace1], layout=layout)\n",
        "fig.show()"
      ],
      "metadata": {
        "id": "K-4I6oGRUHsT"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 2. Item Description (text)"
      ],
      "metadata": {
        "id": "RmVHwGaaXiqQ"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Tokenization ;\n",
        "<br/> text to sentence, sentence to tokens\n",
        "<br/> remove punctuation and stop words\n",
        "<br/> 3 character 이상만"
      ],
      "metadata": {
        "id": "nvvS8rLjfH2b"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import nltk\n",
        "nltk.download('punkt')\n",
        "from nltk.corpus import stopwords\n",
        "from nltk.tokenize import sent_tokenize\n",
        "from nltk.tokenize import word_tokenize"
      ],
      "metadata": {
        "id": "KN_P53_yZSOG"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import re\n",
        "import string"
      ],
      "metadata": {
        "id": "TezCYWKOZ3u8"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "stop = set(stopwords.words('english'))\n",
        "def tokenize(text):\n",
        "  try :\n",
        "    regex = re.compile('[' +re.escape(string.punctuation) + '0-9\\\\r\\\\t\\\\n]')\n",
        "    text = regex.sub(\" \", text) # remove punctuation\n",
        "\n",
        "    tokens_ = [word_tokenize(s) for s in sent_tokenize(text)]\n",
        "    tokens = []\n",
        "    for token_by_sent in tokens_:\n",
        "      tokens += token_by_sent\n",
        "\n",
        "    tokens = list(filter(lambda t: t.lower() not in stop, tokens))\n",
        "    filtered_tokens = [w for w in tokens if re.search('[a-zA-Z]', w)]\n",
        "    filtered_tokens = [w.lower() for w in filtered_tokens if len(w)>=3]\n",
        "\n",
        "    return filtered_tokens\n",
        "\n",
        "  except TypeError as e : print(text, e)"
      ],
      "metadata": {
        "id": "P4zp-jP3YeEy"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "train['tokens'] = train['item_description'].map(tokenize)\n",
        "test['tokens'] = test['item_description'].map(tokenize)"
      ],
      "metadata": {
        "id": "7ov2aaHehtC-"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "for description, tokens in zip(train['item_description'].head(), train['tokens'].head()):\n",
        "  print('description:', description)\n",
        "  print('tokens:', tokens)\n",
        "  print()"
      ],
      "metadata": {
        "id": "ZzVnyDL7htW8"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "general_cats = train['general_cat'].unique()"
      ],
      "metadata": {
        "id": "KV4LSB_plUp5"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from collections import Counter"
      ],
      "metadata": {
        "id": "0ZtTV_1bmtRr"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "cat_desc = dict()\n",
        "for cat in general_cats:\n",
        "  try:\n",
        "    text = \" \".join(train.loc[train['general_cat']==cat, 'item_description'].values)\n",
        "    cat_desc[cat] = tokenize(text)\n",
        "  except TypeError as e : print(e)\n",
        "\n",
        "women100 = Counter(cat_desc['Women']).most_common(100)\n",
        "beauty100 = Counter(cat_desc['Beauty']).most_common(100)\n",
        "kids100 = Counter(cat_desc['Kids']).most_common(100)\n",
        "electronics100 = Counter(cat_desc['Electronics']).most_common(100)"
      ],
      "metadata": {
        "id": "mfmYEqI9jh_K"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "train.loc[train['general_cat']=='Beauty', 'tokens'].values"
      ],
      "metadata": {
        "id": "BSi_L7CbqRWQ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "cat_desc['Beauty']"
      ],
      "metadata": {
        "id": "hQw14-tBpYZK"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from wordcloud import WordCloud"
      ],
      "metadata": {
        "id": "r6Z1zzt0k_d8"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def generate_wordcloud(tup):\n",
        "    wordcloud = WordCloud(background_color='white',\n",
        "                          max_words=50, max_font_size=40,\n",
        "                          random_state=42\n",
        "                         ).generate(str(tup))\n",
        "    return wordcloud"
      ],
      "metadata": {
        "id": "vlTUjz1dkzOV"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "vectorizer = TfidfVectorizer(min_df=10,\n",
        "                             max_features=180000,\n",
        "                             tokenizer=tokenize,\n",
        "                             ngram_range=(1, 2))"
      ],
      "metadata": {
        "id": "Kurci8XUr7JJ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "all_desc = np.append(train['item_description'].values, test['item_description'].values)\n",
        "vz = vectorizer.fit_transform(list(all_desc))"
      ],
      "metadata": {
        "id": "fNrEN7-nr9-8"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#  create a dictionary mapping the tokens to their tfidf values\n",
        "tfidf = dict(zip(vectorizer.get_feature_names(), vectorizer.idf_))\n",
        "tfidf = pd.DataFrame(columns=['tfidf']).from_dict(\n",
        "                    dict(tfidf), orient='index')\n",
        "tfidf.columns = ['tfidf']"
      ],
      "metadata": {
        "id": "sotgPF9Or_ZM"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "tfidf.sort_values(by=['tfidf'], ascending=True).head(10)"
      ],
      "metadata": {
        "id": "xk0BKQ_CsAxy"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "tfidf.sort_values(by=['tfidf'], ascending=False).head(10)"
      ],
      "metadata": {
        "id": "_P0auik7sCHj"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "trn = train.copy()\n",
        "tst = test.copy()\n",
        "trn['is_train'] = 1\n",
        "tst['is_train'] = 0\n",
        "\n",
        "sample_sz = 15000\n",
        "\n",
        "combined_df = pd.concat([trn, tst])\n",
        "combined_sample = combined_df.sample(n=sample_sz)\n",
        "vz_sample = vectorizer.fit_transform(list(combined_sample['item_description']))"
      ],
      "metadata": {
        "id": "rsX2OxO3sD1a"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.decomposition import TruncatedSVD\n",
        "\n",
        "n_comp=30\n",
        "svd = TruncatedSVD(n_components=n_comp, random_state=42)\n",
        "svd_tfidf = svd.fit_transform(vz_sample)"
      ],
      "metadata": {
        "id": "_tK48c5SsGSm"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.manifold import TSNE\n",
        "tsne_model = TSNE(n_components=2, verbose=1, random_state=42, n_iter=500)\n",
        "tsne_tfidf = tsne_model.fit_transform(svd_tfidf)"
      ],
      "metadata": {
        "id": "09cXoRW8sH03"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "output_notebook()\n",
        "plot_tfidf = bp.figure(plot_width=700, plot_height=600,\n",
        "                       title=\"tf-idf clustering of the item description\",\n",
        "    tools=\"pan,wheel_zoom,box_zoom,reset,hover,previewsave\",\n",
        "    x_axis_type=None, y_axis_type=None, min_border=1)"
      ],
      "metadata": {
        "id": "NQz3dw5bsJUN"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "tfidf_df = pd.DataFrame(tsne_tfidf, columns=['x', 'y'])\n",
        "tfidf_df['description'] = combined_sample['item_description']\n",
        "tfidf_df['tokens'] = combined_sample['tokens']\n",
        "tfidf_df['category'] = combined_sample['general_cat']"
      ],
      "metadata": {
        "id": "AtxDKbu3sK11"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "plot_tfidf.scatter(x='x', y='y', source=tfidf_df, alpha=0.7)\n",
        "hover = plot_tfidf.select(dict(type=HoverTool))\n",
        "hover.tooltips={\"description\": \"@description\", \"tokens\": \"@tokens\", \"category\":\"@category\"}\n",
        "show(plot_tfidf)\n"
      ],
      "metadata": {
        "id": "dBa-Pz1asLaG"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.cluster import MiniBatchKMeans\n",
        "\n",
        "num_clusters = 30 # need to be selected wisely\n",
        "kmeans_model = MiniBatchKMeans(n_clusters=num_clusters,\n",
        "                               init='k-means++',\n",
        "                               n_init=1,\n",
        "                               init_size=1000, batch_size=1000, verbose=0, max_iter=1000)"
      ],
      "metadata": {
        "id": "lU55XMNzsOL7"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "kmeans = kmeans_model.fit(vz)\n",
        "kmeans_clusters = kmeans.predict(vz)\n",
        "kmeans_distances = kmeans.transform(vz)"
      ],
      "metadata": {
        "id": "FaMZB5jbsPkh"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "sorted_centroids = kmeans.cluster_centers_.argsort()[:, ::-1]\n",
        "terms = vectorizer.get_feature_names()\n",
        "\n",
        "for i in range(num_clusters):\n",
        "    print(\"Cluster %d:\" % i)\n",
        "    aux = ''\n",
        "    for j in sorted_centroids[i, :10]:\n",
        "        aux += terms[j] + ' | '\n",
        "    print(aux)\n",
        "    print() "
      ],
      "metadata": {
        "id": "ppyNdeKisRd8"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# repeat the same steps for the sample\n",
        "kmeans = kmeans_model.fit(vz_sample)\n",
        "kmeans_clusters = kmeans.predict(vz_sample)\n",
        "kmeans_distances = kmeans.transform(vz_sample)\n",
        "# reduce dimension to 2 using tsne\n",
        "tsne_kmeans = tsne_model.fit_transform(kmeans_distances)"
      ],
      "metadata": {
        "id": "ifVj8bNbsSIH"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#combined_sample.reset_index(drop=True, inplace=True)\n",
        "kmeans_df = pd.DataFrame(tsne_kmeans, columns=['x', 'y'])\n",
        "kmeans_df['cluster'] = kmeans_clusters\n",
        "kmeans_df['description'] = combined_sample['item_description']\n",
        "kmeans_df['category'] = combined_sample['general_cat']\n",
        "#kmeans_df['cluster']=kmeans_df.cluster.astype(str).astype('category')"
      ],
      "metadata": {
        "id": "0pi0OtITsTvm"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "plot_kmeans = bp.figure(plot_width=700, plot_height=600,\n",
        "                        title=\"KMeans clustering of the description\",\n",
        "    tools=\"pan,wheel_zoom,box_zoom,reset,hover,previewsave\",\n",
        "    x_axis_type=None, y_axis_type=None, min_border=1)"
      ],
      "metadata": {
        "id": "xQbBAWD1sWqR"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "source = ColumnDataSource(data=dict(x=kmeans_df['x'], y=kmeans_df['y'],\n",
        "                                    color=colormap[kmeans_clusters],\n",
        "                                    description=kmeans_df['description'],\n",
        "                                    category=kmeans_df['category'],\n",
        "                                    cluster=kmeans_df['cluster']))\n",
        "\n",
        "plot_kmeans.scatter(x='x', y='y', color='color', source=source)\n",
        "hover = plot_kmeans.select(dict(type=HoverTool))\n",
        "hover.tooltips={\"description\": \"@description\", \"category\": \"@category\", \"cluster\":\"@cluster\" }\n",
        "show(plot_kmeans)"
      ],
      "metadata": {
        "id": "TmvhkA0OsYJS"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "cvectorizer = CountVectorizer(min_df=4,\n",
        "                              max_features=180000,\n",
        "                              tokenizer=tokenize,\n",
        "                              ngram_range=(1,2))"
      ],
      "metadata": {
        "id": "IPvhRtN1sZCA"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "cvz = cvectorizer.fit_transform(combined_sample['item_description'])\n",
        "lda_model = LatentDirichletAllocation(n_components=20,\n",
        "                                      learning_method='online',\n",
        "                                      max_iter=20,\n",
        "                                      random_state=42)\n",
        "X_topics = lda_model.fit_transform(cvz)"
      ],
      "metadata": {
        "id": "8ClX1VmWsbSM"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "n_top_words = 10\n",
        "topic_summaries = []\n",
        "\n",
        "topic_word = lda_model.components_  # get the topic words\n",
        "vocab = cvectorizer.get_feature_names()\n",
        "\n",
        "for i, topic_dist in enumerate(topic_word):\n",
        "    topic_words = np.array(vocab)[np.argsort(topic_dist)][:-(n_top_words+1):-1]\n",
        "    topic_summaries.append(' '.join(topic_words))\n",
        "    print('Topic {}: {}'.format(i, ' | '.join(topic_words)))"
      ],
      "metadata": {
        "id": "rFqpSUKfsdLI"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "tsne_lda = tsne_model.fit_transform(X_topics)"
      ],
      "metadata": {
        "id": "JQx1wsrNsglo"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "unnormalized = np.matrix(X_topics)\n",
        "doc_topic = unnormalized/unnormalized.sum(axis=1)\n",
        "\n",
        "lda_keys = []\n",
        "for i, tweet in enumerate(combined_sample['item_description']):\n",
        "    lda_keys += [doc_topic[i].argmax()]\n",
        "\n",
        "lda_df = pd.DataFrame(tsne_lda, columns=['x','y'])\n",
        "lda_df['description'] = combined_sample['item_description']\n",
        "lda_df['category'] = combined_sample['general_cat']\n",
        "lda_df['topic'] = lda_keys\n",
        "lda_df['topic'] = lda_df['topic'].map(int)"
      ],
      "metadata": {
        "id": "zXxZA_LCsh1b"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "source = ColumnDataSource(data=dict(x=lda_df['x'], y=lda_df['y'],\n",
        "                                    color=colormap[lda_keys],\n",
        "                                    description=lda_df['description'],\n",
        "                                    topic=lda_df['topic'],\n",
        "                                    category=lda_df['category']))\n",
        "\n",
        "plot_lda.scatter(source=source, x='x', y='y', color='color')\n",
        "hover = plot_kmeans.select(dict(type=HoverTool))\n",
        "hover = plot_lda.select(dict(type=HoverTool))\n",
        "hover.tooltips={\"description\":\"@description\",\n",
        "                \"topic\":\"@topic\", \"category\":\"@category\"}\n",
        "show(plot_lda)"
      ],
      "metadata": {
        "id": "eJHao-jZsjfS"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def prepareLDAData():\n",
        "    data = {\n",
        "        'vocab': vocab,\n",
        "        'doc_topic_dists': doc_topic,\n",
        "        'doc_lengths': list(lda_df['len_docs']),\n",
        "        'term_frequency':cvectorizer.vocabulary_,\n",
        "        'topic_term_dists': lda_model.components_\n",
        "    } \n",
        "    return data"
      ],
      "metadata": {
        "id": "rEeCNSHbslPJ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import pyLDAvis\n",
        "\n",
        "lda_df['len_docs'] = combined_sample['tokens'].map(len)\n",
        "ldadata = prepareLDAData()\n",
        "pyLDAvis.enable_notebook()\n",
        "prepared_data = pyLDAvis.prepare(**ldadata)"
      ],
      "metadata": {
        "id": "zcE_wC-ysmzu"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}