vis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import pickle\n",
    "import utils\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_name = \"results/analysis-crypto-tweets-processed.csv\"\n",
    "file_test = \"dataset/crypto-tweets-processed.csv\"\n",
    "save_to = \"results/crypto-predicted.csv\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_pred = pd.read_csv(file_name, \n",
    "                      names=[\"id\",\n",
    "                            \"prediction\",\n",
    "                            \"sentiment\",\n",
    "                            \"text\"],\n",
    "                      skiprows=[0])\n",
    "\n",
    "df_test = pd.read_csv(file_test, header=None)\n",
    "df_pred['sentiment'] = np.where(df_pred['prediction'] == 0, 'negative', 'positive')\n",
    "df_pred['text'] = df_test[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>prediction</th>\n",
       "      <th>sentiment</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>positive</td>\n",
       "      <td>b USER_MENTION less than hours crypto USER_MEN...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>negative</td>\n",
       "      <td>bas promised cloakcoin internal security audit...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>positive</td>\n",
       "      <td>b USER_MENTION forget bitcoin its blockchain t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>positive</td>\n",
       "      <td>b USER_MENTION what is aelf elf and what is dr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>positive</td>\n",
       "      <td>b USER_MENTION check out our revised whitepape...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>positive</td>\n",
       "      <td>b USER_MENTION dogezer will be presented at wo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>positive</td>\n",
       "      <td>announced that it will joint venture with cham...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>positive</td>\n",
       "      <td>bdid you invest in myriad xmy its listed on bi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>negative</td>\n",
       "      <td>the bankpire strike back ripple rapidly closin...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>positive</td>\n",
       "      <td>b USER_MENTION forget bitcoin its blockchain t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>positive</td>\n",
       "      <td>b USER_MENTION do you have crypto and a smartp...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>11</td>\n",
       "      <td>1</td>\n",
       "      <td>positive</td>\n",
       "      <td>b USER_MENTION we are doing a huge vip members...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>positive</td>\n",
       "      <td>announced that it will joint venture with cham...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>13</td>\n",
       "      <td>1</td>\n",
       "      <td>positive</td>\n",
       "      <td>stc for sign here URL bitcoin ethereum URL</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>14</td>\n",
       "      <td>1</td>\n",
       "      <td>positive</td>\n",
       "      <td>bUSER_MENTION just gained respect for you the ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>positive</td>\n",
       "      <td>b USER_MENTION bitbay community update jan URL...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>16</td>\n",
       "      <td>1</td>\n",
       "      <td>positive</td>\n",
       "      <td>bon jan artemine token arte released the genes...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>17</td>\n",
       "      <td>1</td>\n",
       "      <td>positive</td>\n",
       "      <td>announced that it will joint venture with cham...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>18</td>\n",
       "      <td>1</td>\n",
       "      <td>positive</td>\n",
       "      <td>b USER_MENTION in addition to the great team b...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>19</td>\n",
       "      <td>1</td>\n",
       "      <td>positive</td>\n",
       "      <td>bredvolution redv is an awesome cryptoproject ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>20</td>\n",
       "      <td>0</td>\n",
       "      <td>negative</td>\n",
       "      <td>b USER_MENTION russia developing system to ide...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>21</td>\n",
       "      <td>1</td>\n",
       "      <td>positive</td>\n",
       "      <td>b USER_MENTION dear bounty members remember yo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>22</td>\n",
       "      <td>1</td>\n",
       "      <td>positive</td>\n",
       "      <td>b USER_MENTION the whitelist is now open to UR...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>23</td>\n",
       "      <td>1</td>\n",
       "      <td>positive</td>\n",
       "      <td>bUSER_MENTION USER_MENTION USER_MENTION USER_M...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>24</td>\n",
       "      <td>0</td>\n",
       "      <td>negative</td>\n",
       "      <td>b USER_MENTION every one of these books in kor...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>25</td>\n",
       "      <td>0</td>\n",
       "      <td>negative</td>\n",
       "      <td>bUSER_MENTION USER_MENTION what is the rate pe...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>26</td>\n",
       "      <td>1</td>\n",
       "      <td>positive</td>\n",
       "      <td>bthe funny yet true story of how our staff wri...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>27</td>\n",
       "      <td>0</td>\n",
       "      <td>negative</td>\n",
       "      <td>bUSER_MENTION they did this weeks ago except l...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>28</td>\n",
       "      <td>1</td>\n",
       "      <td>positive</td>\n",
       "      <td>bUSER_MENTION USER_MENTION nice</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>29</td>\n",
       "      <td>0</td>\n",
       "      <td>negative</td>\n",
       "      <td>bUSER_MENTION USER_MENTION but the price still...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>30</td>\n",
       "      <td>1</td>\n",
       "      <td>positive</td>\n",
       "      <td>b cryptocurrency is already seeping into agric...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>31</td>\n",
       "      <td>0</td>\n",
       "      <td>negative</td>\n",
       "      <td>b USER_MENTION miss a chance to earn with debi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>32</td>\n",
       "      <td>1</td>\n",
       "      <td>positive</td>\n",
       "      <td>bthe launch of web platform URL newsoftheweek ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>33</td>\n",
       "      <td>1</td>\n",
       "      <td>positive</td>\n",
       "      <td>bready for this weeks tip to research and out ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>34</td>\n",
       "      <td>1</td>\n",
       "      <td>positive</td>\n",
       "      <td>important to remember that this is the essenti...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>35</td>\n",
       "      <td>0</td>\n",
       "      <td>negative</td>\n",
       "      <td>b USER_MENTION ethereum is the only crypto wit...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>36</td>\n",
       "      <td>0</td>\n",
       "      <td>negative</td>\n",
       "      <td>b USER_MENTION see the news report by cnbc cry...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>37</td>\n",
       "      <td>1</td>\n",
       "      <td>positive</td>\n",
       "      <td>your trading to the crypto exchange today a di...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>38</td>\n",
       "      <td>1</td>\n",
       "      <td>positive</td>\n",
       "      <td>b USER_MENTION in addition to the great team b...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>39</td>\n",
       "      <td>0</td>\n",
       "      <td>negative</td>\n",
       "      <td>b USER_MENTION ethereum is the only crypto wit...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    id  prediction sentiment  \\\n",
       "0    0           1  positive   \n",
       "1    1           0  negative   \n",
       "2    2           1  positive   \n",
       "3    3           1  positive   \n",
       "4    4           1  positive   \n",
       "5    5           1  positive   \n",
       "6    6           1  positive   \n",
       "7    7           1  positive   \n",
       "8    8           0  negative   \n",
       "9    9           1  positive   \n",
       "10  10           1  positive   \n",
       "11  11           1  positive   \n",
       "12  12           1  positive   \n",
       "13  13           1  positive   \n",
       "14  14           1  positive   \n",
       "15  15           1  positive   \n",
       "16  16           1  positive   \n",
       "17  17           1  positive   \n",
       "18  18           1  positive   \n",
       "19  19           1  positive   \n",
       "20  20           0  negative   \n",
       "21  21           1  positive   \n",
       "22  22           1  positive   \n",
       "23  23           1  positive   \n",
       "24  24           0  negative   \n",
       "25  25           0  negative   \n",
       "26  26           1  positive   \n",
       "27  27           0  negative   \n",
       "28  28           1  positive   \n",
       "29  29           0  negative   \n",
       "30  30           1  positive   \n",
       "31  31           0  negative   \n",
       "32  32           1  positive   \n",
       "33  33           1  positive   \n",
       "34  34           1  positive   \n",
       "35  35           0  negative   \n",
       "36  36           0  negative   \n",
       "37  37           1  positive   \n",
       "38  38           1  positive   \n",
       "39  39           0  negative   \n",
       "\n",
       "                                                 text  \n",
       "0   b USER_MENTION less than hours crypto USER_MEN...  \n",
       "1   bas promised cloakcoin internal security audit...  \n",
       "2   b USER_MENTION forget bitcoin its blockchain t...  \n",
       "3   b USER_MENTION what is aelf elf and what is dr...  \n",
       "4   b USER_MENTION check out our revised whitepape...  \n",
       "5   b USER_MENTION dogezer will be presented at wo...  \n",
       "6   announced that it will joint venture with cham...  \n",
       "7   bdid you invest in myriad xmy its listed on bi...  \n",
       "8   the bankpire strike back ripple rapidly closin...  \n",
       "9   b USER_MENTION forget bitcoin its blockchain t...  \n",
       "10  b USER_MENTION do you have crypto and a smartp...  \n",
       "11  b USER_MENTION we are doing a huge vip members...  \n",
       "12  announced that it will joint venture with cham...  \n",
       "13         stc for sign here URL bitcoin ethereum URL  \n",
       "14  bUSER_MENTION just gained respect for you the ...  \n",
       "15  b USER_MENTION bitbay community update jan URL...  \n",
       "16  bon jan artemine token arte released the genes...  \n",
       "17  announced that it will joint venture with cham...  \n",
       "18  b USER_MENTION in addition to the great team b...  \n",
       "19  bredvolution redv is an awesome cryptoproject ...  \n",
       "20  b USER_MENTION russia developing system to ide...  \n",
       "21  b USER_MENTION dear bounty members remember yo...  \n",
       "22  b USER_MENTION the whitelist is now open to UR...  \n",
       "23  bUSER_MENTION USER_MENTION USER_MENTION USER_M...  \n",
       "24  b USER_MENTION every one of these books in kor...  \n",
       "25  bUSER_MENTION USER_MENTION what is the rate pe...  \n",
       "26  bthe funny yet true story of how our staff wri...  \n",
       "27  bUSER_MENTION they did this weeks ago except l...  \n",
       "28                    bUSER_MENTION USER_MENTION nice  \n",
       "29  bUSER_MENTION USER_MENTION but the price still...  \n",
       "30  b cryptocurrency is already seeping into agric...  \n",
       "31  b USER_MENTION miss a chance to earn with debi...  \n",
       "32  bthe launch of web platform URL newsoftheweek ...  \n",
       "33  bready for this weeks tip to research and out ...  \n",
       "34  important to remember that this is the essenti...  \n",
       "35  b USER_MENTION ethereum is the only crypto wit...  \n",
       "36  b USER_MENTION see the news report by cnbc cry...  \n",
       "37  your trading to the crypto exchange today a di...  \n",
       "38  b USER_MENTION in addition to the great team b...  \n",
       "39  b USER_MENTION ethereum is the only crypto wit...  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_pred.head(40)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x113b69320>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAEiCAYAAADptCm5AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAEZlJREFUeJzt3XuspHV9x/H3BxYrqCAa2BWWi6KC9YZoFGKjU0kFsUKtQNXSoLWXiFVrjQrWlGO1NWvjNdYLlVC0AkKxBROjVGVqUFdiAZGr1AtXWUth0ZrWIHz7xwybw3LO7vHMzHnO/Ob9Sk52nt/zPPP7hnnOh9/5PZdJVSFJmn47dF2AJGk8DHRJaoSBLkmNMNAlqREGuiQ1wkCXpEZsN9CTnJ5kU5Ir57W9N8m1Sa5Icn6SXeetOyXJDcP1L5xU4ZKkB1rKCP0M4Iit2i4CnlxVBwM3AKcAJPl14HjgScCLgI8myfjKlSQtZruBXlWXAHdt1fblqrpvuLgRWD98fTRwTlX9sqp+xCDsnz2+ciVJixnHHPofAl8Yvt4buHneuluHbZKkCRsp0JP8JXBPVZ19f9MCm/lsAUlaAWuWu2OSE4GjgBfMa74F2Gfe8nrgtkX2N+glaRmqasFzk0sN9DBv9J3kSOCtwPOq6hfztrsQ+EySDzCYank8cOk2ilpi99qeubk55ubmui5DehCPzfHa1nUm2w30JGcBPeDRSW4CTgXeDjwE+Lfhm2+sqpOq6pok5wLXAPcAJ5WpLUkrYruBXlWvXKD5jG1s/x7gPaMUJUn61XmnaCN6vV7XJUgL8thcOelqRiSJszGS9CtKsuhJUUfoktQIA12SGmGgS1IjDHRJaoSBLkmNWPat/7Ni3br92bTpxq7LaMbatftx++0/6roMqUletrgdgzthV3+d0yM+8kEagZctStIMMNAlqREGuiQ1wkCXpEYY6JLUCANdkhphoEtSIwx0SWqEgS5JjTDQJakRBrokNcJAl6RGGOiS1AgDXZIaYaBLUiMMdElqhIEuSY0w0CWpEQa6JDViu4Ge5PQkm5JcOa9t9yQXJbk+yZeS7DZv3YeT3JDkiiQHT6pwSdIDLWWEfgZwxFZtJwNfrqoDga8CpwAkeRFwQFU9AfhT4ONjrFWStA3bDfSqugS4a6vmY4Azh6/PHC7f3/6p4X7fAnZLsnY8pUqStmW5c+h7VtUmgKq6Hdhz2L43cPO87W4dtkmSJmzcJ0WzQFuNuQ9J0gLWLHO/TUnWVtWmJOuAnwzbbwH2mbfdeuC2xd5kbm5uy+ter0ev11tmOZLUpn6/T7/fX9K2qdr+ADrJ/sDnq+qpw+UNwJ1VtSHJycAjq+rkJEcBr6uqFyc5FPhgVR26yHvWUvruWhL8I2OcwjR87tJqlYSqWmg2ZPuBnuQsoAc8GtgEnAr8K3Aeg9H4TcBxVbV5uP1HgCOBnwOvrqrLFnlfA30mGejSKEYK9Ekx0GeVgS6NYluB7p2iktQIA12SGmGgS1IjDHRJaoSBLkmNMNAlqREGuiQ1wkCXpEYY6JLUCANdkhphoEtSIwx0SWqEgS5JjTDQJakRBrokNcJAl6RGGOiS1AgDXZIaYaBLUiMMdElqhIEuSY0w0CWpEQa6JDXCQJekRhjoktQIA12SGmGgS1IjDHRJaoSBLkmNGCnQk7wpyVVJrkzymSQPSbJ/ko1Jrk9ydpI14ypWkrS4ZQd6kr2A1wOHVNXTgDXAK4ANwPuq6kBgM/CacRQqSdq2UadcdgQeNhyF7wzcBvwmcP5w/ZnAS0fsQ5K0BMsO9Kq6DXgfcBNwK3A3cBmwuaruG252C7DXqEVKkrZv2fPbSR4JHAPsxyDMzwNetMCmtdh7zM3NbXnd6/Xo9XrLLUeSmtTv9+n3+0vaNlWL5u22d0yOBY6oqj8eLv8BcBhwLLCuqu5LcihwalU9KOiT1HL7XklJ2Mb/k/QrC9PwuUurVRKqKgutG2UO/Sbg0CQPzSD1DgeuBi4GjhtucyJwwQh9SJKWaNkjdIAkpwIvB+4BLgf+CFgPnAPsPmw7oaruWWBfR+gzyRG6NIptjdBHCvRRGOizykCXRjGpKRdJ0ipioEtSIwx0SWqEgS5JjTDQJakRBrokNcJAl6RGGOiS1AgDXZIaYaBLUiMMdElqhIEuSY0w0CWpEQa6JDXCQJekRhjoktQIA12SGmGgS1IjDHRJaoSBLkmNMNAlqREGuiQ1wkCXpEYY6JLUCANdkhphoEtSIwx0SWqEgS5JjRgp0JPsluS8JNcmuTrJc5LsnuSiJNcn+VKS3cZVrCRpcaOO0D8EfKGqngQ8HbgOOBn4clUdCHwVOGXEPiRJS5CqWt6OySOAK6rqgK3arwOeX1WbkqwD+lV10AL713L7XklJgNVf5/QI0/C5S6tVEqoqC60bZYT+OOCOJGckuSzJaUl2AdZW1SaAqrod2GOEPiRJS7RmxH0PAV5XVd9O8gEG0y1LHn7Nzc1ted3r9ej1eiOUI0nt6ff79Pv9JW07ypTLWuCbVfW44fJvMAj0A4DevCmXi4dz7Fvv75TLTHLKRRrFRKZchtMqNyd54rDpcOBq4ELgVcO2E4ELltuHJGnplj1CB0jydOCTwE7AD4BXAzsC5wL7ADcBx1XV5gX2dYQ+kxyhS6PY1gh9pEAfhYE+qwx0aRSTuspFkrSKGOiS1AgDXZIaYaBLUiMMdElqhIEuSY0w0CWpEQa6JDXCQJekRhjoktQIA12SGmGgS1IjDHRJaoSBLkmNMNAlqREGuiQ1wkCXpEYY6JLUCANdkhphoEtSIwx0SWqEgS5JjTDQJakRBrokNcJAl6RGGOiS1AgDXZIaYaBLUiNGDvQkOyS5LMmFw+X9k2xMcn2Ss5OsGb1MSdL2jGOE/kbgmnnLG4D3VdWBwGbgNWPoQ5K0HSMFepL1wFHAJ+c1vwA4f/j6TOClo/QhSVqaUUfoHwDeAhRAkkcDd1XVfcP1twB7jdiHJGkJlh3oSV4MbKqqK4Dc3zzv9f1quX1IkpZulBOWzwWOTnIUsDPwCOCDwG5JdhiO0tcDty32BnNzc1te93o9er3eCOVIUnv6/T79fn9J26Zq9AF0kucDb66qo5N8FvhcVX02yceA71TVxxfYp8bR96QlwT8yxilMw+curVZJqKqtZ0KA0UboizkZOCfJu4DLgdMn0Ic089at259Nm27suoxmrF27H7ff/qOuyxjJWEboy+rYEfqMcoQ+Lh6b4zYdx+a2RujeKSpJjTDQJakRBrokNcJAl6RGGOiS1AgDXZIaYaBLUiMMdElqhIEuSY0w0CWpEQa6JDXCQJekRhjoktQIA12SGmGgS1IjDHRJaoSBLkmNMNAlqREGuiQ1wkCXpEYY6JLUCANdkhphoEtSIwx0SWqEgS5JjTDQJakRBrokNcJAl6RGLDvQk6xP8tUk1yT5bpI3DNt3T3JRkuuTfCnJbuMrV5K0mFTV8nZM1gHrquqKJA8H/gM4Bng18N9V9d4kbwN2r6qTF9i/ltv3SkoCrP46p0eYhs99Gnhsjtt0HJtJqKostG7ZI/Squr2qrhi+/h/gWmA9g1A/c7jZmcDvLLcPSdLSjWUOPcn+wMHARmBtVW2CQegDe4yjD0nSto0c6MPpln8G3jgcqa/+v1kkqUFrRtk5yRoGYf7pqrpg2Lwpydqq2jScZ//JYvvPzc1ted3r9ej1eqOUI0nN6ff79Pv9JW277JOiAEk+BdxRVX8xr20DcGdVbfCkqB5sOk48TQOPzXGbjmNzWydFR7nK5bnA14DvMjiqCng7cClwLrAPcBNwXFVtXmB/A30mTccvzTTw2By36Tg2JxLoozLQZ9V0/NJMA4/NcZuOY3Mily1KklYXA12SGmGgS1IjDHRJaoSBLkmNMNAlqREGuiQ1wkCXpEYY6JLUCANdkhphoEtSIwx0SWqEgS5JjTDQJakRBrokNcJAl6RGGOiS1AgDXZIaYaBLUiMMdElqhIEuSY0w0CWpEQa6JDXCQJekRhjoktQIA12SGmGgS1IjDHRJasTEAj3JkUmuS/K9JG+bVD+SpIGJBHqSHYCPAEcATwZekeSgSfSl+/W7LkBaRL/rAmbGpEbozwZuqKobq+oe4BzgmAn1JcBfGq1e/a4LmBmTCvS9gZvnLd8ybJMkTcikAj0LtNWE+pIkAWsm9L63APvOW14P3Lb1RslCub8aTUud7+y6gCWZns99GkzLf0uPzZWQqvEPnJPsCFwPHA78GLgUeEVVXTv2ziRJwIRG6FV1b5I/Ay5iMK1zumEuSZM1kRG6JGnleaeoJDXCQJekRhjokiYiyc5JDuy6jllioE+xDJyQ5K+Gy/smeXbXdUlJXgJcAXxxuHxwkgu7rap9Bvp0+yhwGPCK4fLPgL/vrhxpizkGjwDZDFBVVwD7d1jPTJjUjUVaGc+pqkOSXA5QVXcleUjXRUnAL6vq7mm/UWfaGOjT7Z7hTVwFkGQP4L5uS5IAuCrJK4EdkzwBeAPwjY5rap5TLtPtw8C/AHsm+RvgEuBvuy1JAuD1DB6d/QvgLOBu4M87rWgGeGPRlBs+Z/5wBg/1+Ip35Go1SPKMqrq86zpmjYE+xZJ8CPhsVfmnrFaVJBcDjwHOA86pqqs7LmkmOOUy3S4D3pHkP5P8XZJndV2QBFBVvwn0gP8CTkvy3STv6Laq9jlCb0CSRwEvA14O7FtVT+i4JGmLJE8F3gr8XlV5FdYEOUJvw+OBgxhc53tdt6VIkORJSeaSXMXg+4W/weB7ETRBjtCnWJINwO8C3wfOBT5XVZu7rUqCJBuBs4HzqupBX26jyfA69On2Q+Cwqrqj60Kk+arq0K5rmEWO0KdQkoOq6rokhyy0vqouW+maJIAk51bV8Um+ywO/RzhAVdXTOiptJhjoUyjJaVX1J8NLw7ZWVfWCFS9KApI8pqp+nGS/hdZX1Y0rXdMsMdCnWJKHVtX/ba9NWmlJNlTV27bXpvHyKpfpttANRd5kpNXgtxZoe9GKVzFjPCk6hZKsA/YGdk7yDAbzkwC7Art0VphmXpLXAicBj0ty5bxVjwC+3k1Vs8MplymU5ETgVcCzgG/PW/Uz4B+r6nNd1CUl2Q3YHXgPcPK8VT+rqju7qWp2GOhTLMnLqur8ruuQFpNkT+Ch9y9X1U0dltM8A30KJTmhqv4pyZt54KVhAFTV+zsoS9pi+BV07wf2An4C7AdcW1VP7rSwxnlSdDo9bPjvwxnMTW79I3Xt3cChwPeq6rEMHvHsHPqEOUKXNHZJvl1Vz0ryHeAZVXVfkkuryi8xnyBH6FMsyXuT7JpkpyRfSXJHkhO6rksCNid5OPA14DPDZ/f/suOammegT7cXVtVPgd8GbgGeCLyl25IkAI4B/hd4E/BFBg+Qe0mnFc0Ar0OfbjsN/z0KOLuq7vRb1rUaVNXP5y2e2VkhM8ZAn26fT3Idg5HQSUn2ALztX51L8jMefAXW3Qzum3hzVf1g5atqnydFp1yS3YGfVtW9SXYBdq2q27uuS7MtyTuB24CzGNzJ/HJgHXA98Nqq6nVXXbsM9CmWZCfgtcDzhk3/Dny8qu7prioJknyrqp6zVdvGqjo0yXeq6uld1dYyT4pOt48BzwQ+Ovw5ZNgmde2+JMcn2WH4c/y8dY4iJ8QR+hRbaKTj6EerQZLHAR8CDmMQ4BsZXPFyK/DMqrqkw/Ka5UnR6XZvkgOq6vuw5Zfo3o5rkhie9FzsMkXDfEIM9On2FuDiJPdfMbA/8OruypEGkjyRwfTf2qp6SpKnAUdX1bs7Lq1pzqFPt68DnwDuG/58AvhmpxVJA/8AnALcA1BVVzK40kUTZKBPt08BjwXeNfx5LPDpTiuSBnapqku3avPW/wlzymW6HbjVCdCLhw9Dkrp2R5IDGF7RkuRY4MfdltQ+A326XZ7k0KraCJDkOfiIUq0OrwNOAw5KcivwQ+D3uy2pfV62OMWSXAscCNz/LTD7AtcymE+vqnpaV7VptiX5NeBYBifqHwX8lMEx+ddd1tU6R+jT7ciuC5AWcQGwGbiMwSMAtAIcoUsauyRXVdVTuq5j1niVi6RJ+EaSp3ZdxKxxhC5p7JJcAzyewcnQXzB44qLndSbMQJc0dkn2W6i9qm5c6VpmiYEuSY1wDl2SGmGgS1IjDHRJaoSBLkmNMNAlqRH/D+SD/9FpqfHQAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x113b69fd0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "df_pred['sentiment'].value_counts().plot(kind='bar')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df_pred.to_csv(path_or_buf=save_to,\n",
    "               sep=',',\n",
    "               columns=['id', 'prediction', 'text'],\n",
    "               header=None,\n",
    "               index=False,\n",
    "               encoding='utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "FREQ_DIST_FILE = 'results/crypto-predicted-freqdist.pkl'\n",
    "BI_FREQ_DIST_FILE = 'results/crypto-predicted-freqdist-bi.pkl'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('b', 107),\n",
       " ('to', 65),\n",
       " ('crypto', 63),\n",
       " ('the', 61),\n",
       " ('and', 34),\n",
       " ('is', 33),\n",
       " ('on', 32),\n",
       " ('of', 29),\n",
       " ('in', 29),\n",
       " ('with', 29),\n",
       " ('a', 25),\n",
       " ('blockchain', 18),\n",
       " ('bitcoin', 18),\n",
       " ('our', 17),\n",
       " ('you', 17),\n",
       " ('cryptocurrency', 17),\n",
       " ('ico', 17),\n",
       " ('that', 16),\n",
       " ('for', 16),\n",
       " ('this', 16)]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open(FREQ_DIST_FILE, 'rb') as pkl_file:\n",
    "    freq_dist = pickle.load(pkl_file)\n",
    "unigrams = freq_dist.most_common(20)\n",
    "unigrams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(('wsj', 'nytimes'), 9),\n",
       " (('crypto', 'blockchain'), 6),\n",
       " (('on', 'our'), 6),\n",
       " (('bmcgraw', 'intends'), 6),\n",
       " (('intends', 'to'), 6),\n",
       " (('to', 'acquire'), 6),\n",
       " (('acquire', 'an'), 6),\n",
       " (('an', 'automotive'), 6),\n",
       " (('automotive', 'dealership'), 6),\n",
       " (('dealership', 'services'), 6),\n",
       " (('services', 'wsj'), 6),\n",
       " (('the', 'world'), 5),\n",
       " (('world', 'crypto'), 5),\n",
       " (('we', 'are'), 5),\n",
       " (('to', 'the'), 5),\n",
       " (('of', 'the'), 5),\n",
       " (('to', 'get'), 5),\n",
       " (('on', 'crypto'), 5),\n",
       " (('with', 'crypto'), 5),\n",
       " (('b', 'forget'), 4)]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open(BI_FREQ_DIST_FILE, 'rb') as pkl_file:\n",
    "    freq_dist = pickle.load(pkl_file)\n",
    "bigrams = freq_dist.most_common(20)\n",
    "bigrams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "poly1d([-0.80910616,  4.36079972])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open(FREQ_DIST_FILE, 'rb') as pkl_file:\n",
    "    freq_dist = pickle.load(pkl_file)\n",
    "unigrams = freq_dist.most_common(100)\n",
    "log_ranks = np.log(range(1, 101))\n",
    "log_freqs = np.log([i[1] for i in unigrams])\n",
    "z = np.polyfit(log_ranks, log_freqs, 1)\n",
    "p = np.poly1d(z)\n",
    "p"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}