FS_annotation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "# Annotating tweets for location extraction and geocoding"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "This notebook is intended for annotating tweets for computing F-score statistics\n",
    "\n",
    "## Dataset structure\n",
    "\n",
    "The tweets dataset are not filtered for a particular topic but cover a range of topics all with different legths. The tweets are all in english language\n",
    "\n",
    "**For F-score test:**  \n",
    "- Determine if a tweet contains a location or not.\n",
    "- If unsure, label the tweet in the class you are more certain of. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-11-19T14:11:07.448284Z",
     "start_time": "2018-11-19T14:11:07.124074Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from IPython.display import display, HTML\n",
    "import json\n",
    "import os"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "## Loading Data\n",
    "\n",
    "Insert your annotator id in the annotator_name variable."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "ename": "FileNotFoundError",
     "evalue": "[Errno 2] No such file or directory: 'df_location_entities1.csv'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-2-e3ac81e7f098>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mpandas\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mpd\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      2\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mnumpy\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 3\u001b[1;33m \u001b[0mdf\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mread_csv\u001b[0m \u001b[1;33m(\u001b[0m\u001b[1;34m'df_location_entities1.csv'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      4\u001b[0m \u001b[0mdf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mto_json\u001b[0m \u001b[1;33m(\u001b[0m\u001b[1;34m'ss_california_tweets.json'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\ESRI\\conda\\envs\\myclone\\lib\\site-packages\\pandas\\io\\parsers.py\u001b[0m in \u001b[0;36mread_csv\u001b[1;34m(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, squeeze, prefix, mangle_dupe_cols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, skipfooter, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, dayfirst, cache_dates, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, doublequote, escapechar, comment, encoding, dialect, error_bad_lines, warn_bad_lines, delim_whitespace, low_memory, memory_map, float_precision)\u001b[0m\n\u001b[0;32m    684\u001b[0m     )\n\u001b[0;32m    685\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 686\u001b[1;33m     \u001b[1;32mreturn\u001b[0m \u001b[0m_read\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkwds\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    687\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    688\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\ESRI\\conda\\envs\\myclone\\lib\\site-packages\\pandas\\io\\parsers.py\u001b[0m in \u001b[0;36m_read\u001b[1;34m(filepath_or_buffer, kwds)\u001b[0m\n\u001b[0;32m    450\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    451\u001b[0m     \u001b[1;31m# Create the parser.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 452\u001b[1;33m     \u001b[0mparser\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mTextFileReader\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfp_or_buf\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    453\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    454\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mchunksize\u001b[0m \u001b[1;32mor\u001b[0m \u001b[0miterator\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\ESRI\\conda\\envs\\myclone\\lib\\site-packages\\pandas\\io\\parsers.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, f, engine, **kwds)\u001b[0m\n\u001b[0;32m    934\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"has_index_names\"\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mkwds\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"has_index_names\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    935\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 936\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_make_engine\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mengine\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    937\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    938\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mclose\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\ESRI\\conda\\envs\\myclone\\lib\\site-packages\\pandas\\io\\parsers.py\u001b[0m in \u001b[0;36m_make_engine\u001b[1;34m(self, engine)\u001b[0m\n\u001b[0;32m   1166\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_make_engine\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mengine\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m\"c\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1167\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mengine\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;34m\"c\"\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1168\u001b[1;33m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_engine\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mCParserWrapper\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mf\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1169\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1170\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mengine\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;34m\"python\"\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\ESRI\\conda\\envs\\myclone\\lib\\site-packages\\pandas\\io\\parsers.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, src, **kwds)\u001b[0m\n\u001b[0;32m   1996\u001b[0m         \u001b[0mkwds\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"usecols\"\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0musecols\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1997\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1998\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_reader\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mparsers\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mTextReader\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msrc\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1999\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0munnamed_cols\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_reader\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0munnamed_cols\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2000\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mpandas\\_libs\\parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader.__cinit__\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;32mpandas\\_libs\\parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader._setup_parser_source\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: 'df_location_entities1.csv'"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "df = pd.read_csv ('df_location_entities1.csv')\n",
    "df.to_json ('ss_california_tweets.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>text</th>\n",
       "      <th>place</th>\n",
       "      <th>long</th>\n",
       "      <th>lat</th>\n",
       "      <th>clean_text</th>\n",
       "      <th>GPE</th>\n",
       "      <th>FAC</th>\n",
       "      <th>ORG</th>\n",
       "      <th>LOC</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>I'm at My Home Gym in Pacifica, CA https://t....</td>\n",
       "      <td>Pacifica; CA</td>\n",
       "      <td>-122.500464</td>\n",
       "      <td>37.593650</td>\n",
       "      <td>Im at My Home Gym in Pacifica, CA</td>\n",
       "      <td>Pacifica, CA</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>_styledbym.e killed it with this #shadowroot #...</td>\n",
       "      <td></td>\n",
       "      <td>-121.989751</td>\n",
       "      <td>38.355840</td>\n",
       "      <td>styledbym.e killed it with this shadowroot col...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>Primigi Classic loafers for your boy or girl. ...</td>\n",
       "      <td>East Oakdale, CA</td>\n",
       "      <td>-120.829960</td>\n",
       "      <td>37.774330</td>\n",
       "      <td>Primigi Classic loafers for your boy or girl. ...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>Primigi</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>Warriors single game tickets go on sale at 10...</td>\n",
       "      <td>San Jose; CA</td>\n",
       "      <td>-121.891766</td>\n",
       "      <td>37.332484</td>\n",
       "      <td>Warriors single game tickets go on sale at 10...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>I'm at Hardly Strictly Bluegrass in San Franc...</td>\n",
       "      <td>San Francisco; CA</td>\n",
       "      <td>-122.489542</td>\n",
       "      <td>37.771727</td>\n",
       "      <td>Im at Hardly Strictly Bluegrass in San Franci...</td>\n",
       "      <td>San Francisco, CA</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1897</th>\n",
       "      <td>1995</td>\n",
       "      <td>how do i change the font on my phone 😐</td>\n",
       "      <td>6fafb06c49df870f</td>\n",
       "      <td>-122.264870</td>\n",
       "      <td>37.998157</td>\n",
       "      <td>how do i change the font on my phone</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1898</th>\n",
       "      <td>1996</td>\n",
       "      <td>@Vanessaah_x I just ran out lol</td>\n",
       "      <td>ee2cfc9feb061a08</td>\n",
       "      <td>-120.845655</td>\n",
       "      <td>37.504062</td>\n",
       "      <td>at Vanessaahx I just ran out lol</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1899</th>\n",
       "      <td>1997</td>\n",
       "      <td>@muhbellsaywhaat loves me. I know she do.</td>\n",
       "      <td>8004d2bebcc13e8c</td>\n",
       "      <td>-122.039433</td>\n",
       "      <td>37.971872</td>\n",
       "      <td>at muhbellsaywhaat loves me. I know she do.</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1900</th>\n",
       "      <td>1998</td>\n",
       "      <td>#ElvistheCorgi #SamtheCorgi #GigitheFrenchie #...</td>\n",
       "      <td></td>\n",
       "      <td>-122.894090</td>\n",
       "      <td>38.479099</td>\n",
       "      <td>ElvistheCorgi SamtheCorgi GigitheFrenchie dog ...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1901</th>\n",
       "      <td>1999</td>\n",
       "      <td>Just posted a photo @ Bay Bridge, San Francis...</td>\n",
       "      <td>San Francisco; CA</td>\n",
       "      <td>-122.405494</td>\n",
       "      <td>37.769230</td>\n",
       "      <td>Just posted a photo at  Bay Bridge, San Franc...</td>\n",
       "      <td>San Francisco</td>\n",
       "      <td>Bay Bridge</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1902 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      Unnamed: 0                                               text  \\\n",
       "0              0   I'm at My Home Gym in Pacifica, CA https://t....   \n",
       "1              1  _styledbym.e killed it with this #shadowroot #...   \n",
       "2              2  Primigi Classic loafers for your boy or girl. ...   \n",
       "3              3   Warriors single game tickets go on sale at 10...   \n",
       "4              4   I'm at Hardly Strictly Bluegrass in San Franc...   \n",
       "...          ...                                                ...   \n",
       "1897        1995             how do i change the font on my phone 😐   \n",
       "1898        1996                    @Vanessaah_x I just ran out lol   \n",
       "1899        1997          @muhbellsaywhaat loves me. I know she do.   \n",
       "1900        1998  #ElvistheCorgi #SamtheCorgi #GigitheFrenchie #...   \n",
       "1901        1999   Just posted a photo @ Bay Bridge, San Francis...   \n",
       "\n",
       "                   place        long        lat  \\\n",
       "0           Pacifica; CA -122.500464  37.593650   \n",
       "1                        -121.989751  38.355840   \n",
       "2       East Oakdale, CA -120.829960  37.774330   \n",
       "3           San Jose; CA -121.891766  37.332484   \n",
       "4      San Francisco; CA -122.489542  37.771727   \n",
       "...                  ...         ...        ...   \n",
       "1897    6fafb06c49df870f -122.264870  37.998157   \n",
       "1898    ee2cfc9feb061a08 -120.845655  37.504062   \n",
       "1899    8004d2bebcc13e8c -122.039433  37.971872   \n",
       "1900                     -122.894090  38.479099   \n",
       "1901   San Francisco; CA -122.405494  37.769230   \n",
       "\n",
       "                                             clean_text                GPE  \\\n",
       "0                     Im at My Home Gym in Pacifica, CA       Pacifica, CA   \n",
       "1     styledbym.e killed it with this shadowroot col...                      \n",
       "2     Primigi Classic loafers for your boy or girl. ...                      \n",
       "3      Warriors single game tickets go on sale at 10...                      \n",
       "4      Im at Hardly Strictly Bluegrass in San Franci...  San Francisco, CA   \n",
       "...                                                 ...                ...   \n",
       "1897              how do i change the font on my phone                       \n",
       "1898                   at Vanessaahx I just ran out lol                      \n",
       "1899        at muhbellsaywhaat loves me. I know she do.                      \n",
       "1900  ElvistheCorgi SamtheCorgi GigitheFrenchie dog ...                      \n",
       "1901   Just posted a photo at  Bay Bridge, San Franc...      San Francisco   \n",
       "\n",
       "             FAC      ORG LOC  \n",
       "0                              \n",
       "1                              \n",
       "2                 Primigi      \n",
       "3                              \n",
       "4                              \n",
       "...          ...      ...  ..  \n",
       "1897                           \n",
       "1898                           \n",
       "1899                           \n",
       "1900                           \n",
       "1901  Bay Bridge               \n",
       "\n",
       "[1902 rows x 10 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df.replace(np.nan, '', regex=True)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-11-19T14:11:07.806330Z",
     "start_time": "2018-11-19T14:11:07.449285Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ss_california_tweets.json\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>clean_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>I'm at My Home Gym in Pacifica, CA https://t....</td>\n",
       "      <td>Im at My Home Gym in Pacifica, CA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>_styledbym.e killed it with this #shadowroot #...</td>\n",
       "      <td>styledbym.e killed it with this shadowroot col...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Primigi Classic loafers for your boy or girl. ...</td>\n",
       "      <td>Primigi Classic loafers for your boy or girl. ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Warriors single game tickets go on sale at 10...</td>\n",
       "      <td>Warriors single game tickets go on sale at 10...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>I'm at Hardly Strictly Bluegrass in San Franc...</td>\n",
       "      <td>Im at Hardly Strictly Bluegrass in San Franci...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text  \\\n",
       "0   I'm at My Home Gym in Pacifica, CA https://t....   \n",
       "1  _styledbym.e killed it with this #shadowroot #...   \n",
       "2  Primigi Classic loafers for your boy or girl. ...   \n",
       "3   Warriors single game tickets go on sale at 10...   \n",
       "4   I'm at Hardly Strictly Bluegrass in San Franc...   \n",
       "\n",
       "                                          clean_text  \n",
       "0                  Im at My Home Gym in Pacifica, CA  \n",
       "1  styledbym.e killed it with this shadowroot col...  \n",
       "2  Primigi Classic loafers for your boy or girl. ...  \n",
       "3   Warriors single game tickets go on sale at 10...  \n",
       "4   Im at Hardly Strictly Bluegrass in San Franci...  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ss_california_tweets.json\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>clean_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>I'm at My Home Gym in Pacifica, CA https://t....</td>\n",
       "      <td>Im at My Home Gym in Pacifica, CA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>_styledbym.e killed it with this #shadowroot #...</td>\n",
       "      <td>styledbym.e killed it with this shadowroot col...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Primigi Classic loafers for your boy or girl. ...</td>\n",
       "      <td>Primigi Classic loafers for your boy or girl. ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Warriors single game tickets go on sale at 10...</td>\n",
       "      <td>Warriors single game tickets go on sale at 10...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>I'm at Hardly Strictly Bluegrass in San Franc...</td>\n",
       "      <td>Im at Hardly Strictly Bluegrass in San Franci...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text  \\\n",
       "0   I'm at My Home Gym in Pacifica, CA https://t....   \n",
       "1  _styledbym.e killed it with this #shadowroot #...   \n",
       "2  Primigi Classic loafers for your boy or girl. ...   \n",
       "3   Warriors single game tickets go on sale at 10...   \n",
       "4   I'm at Hardly Strictly Bluegrass in San Franc...   \n",
       "\n",
       "                                          clean_text  \n",
       "0                  Im at My Home Gym in Pacifica, CA  \n",
       "1  styledbym.e killed it with this shadowroot col...  \n",
       "2  Primigi Classic loafers for your boy or girl. ...  \n",
       "3   Warriors single game tickets go on sale at 10...  \n",
       "4   Im at Hardly Strictly Bluegrass in San Franci...  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "dataset_file_names = ('ss_california_tweets.json','ss_california_tweets.json')\n",
    "\n",
    "# Remember to replace annotator_name with own names\n",
    "annotator_name = 'Helen_FS'\n",
    "\n",
    "for fn in dataset_file_names:\n",
    "    print(fn)\n",
    "    df = pd.DataFrame(json.load(open(fn)))\n",
    "    display(df.head()[['text', 'clean_text']])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "## Annotation\n",
    "\n",
    "### Helper function\n",
    "\n",
    "This function loads the data (using partially annotated .json files if available) and saves it after every annotation.\n",
    "\n",
    "This means that annotation can simply be picked up again whenever desired. Intermediate and final results are saved with the original filename with `_annotated`appended.\n",
    "\n",
    "Only the specified labels (`0,1` by default) are accepted as input, `p` prints a progress bar and any other keys show a help text."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-11-19T14:18:22.336320Z",
     "start_time": "2018-11-19T14:18:22.041078Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "def annotate_tweet_df(fn, possible_labels=('0', '1')):\n",
    "    def process_input(user_input):\n",
    "        if user_input in possible_labels:\n",
    "            return user_input\n",
    "        elif user_input.startswith('p'):\n",
    "            progressbar(compute_annotation_progress(), max_num=len(df))\n",
    "            vc = df[label_column_name].value_counts()\n",
    "            print('labels\\t',  ', '.join([str(k)+': ' + str(v) for k,v in zip(vc.keys(), vc.values)]))\n",
    "        elif user_input.startswith('q'):\n",
    "            raise\n",
    "        else:\n",
    "            print(help_text)\n",
    "\n",
    "        return process_input(input('\\t'))\n",
    "\n",
    "    def compute_annotation_progress():\n",
    "        if label_column_name not in df.keys():\n",
    "            return 0\n",
    "        return len(df) - df[label_column_name].isna().sum()\n",
    "\n",
    "    def progressbar(it, max_num, size=60):\n",
    "        finished = int(round((it / max_num * size))) if it > 0 else 0\n",
    "        rest = size - finished\n",
    "        print('[' + finished * '|' + rest * '.' + ']\\t', it, '/', max_num)\n",
    "\n",
    "    help_text = '\\n'.join(['Possible Commands', str(possible_labels) + '\\tpossible labels',\n",
    "                           'h\\tshow this help', 'p\\tshow progress', 'q\\tquit', ''])\n",
    "\n",
    "    label_column_name = 'label_' + annotator_name\n",
    "    annotated_df_fn = fn.split('.json')[0] + '_annotated' + annotator_name + '.json'\n",
    "\n",
    "    if os.path.exists(annotated_df_fn) and os.path.isfile(annotated_df_fn):\n",
    "        print(annotated_df_fn, 'already exists, continuing previous annotation process')\n",
    "        df: pd.DataFrame = pd.DataFrame(json.load(open(annotated_df_fn)))\n",
    "    else:\n",
    "        df: pd.DataFrame =  pd.DataFrame(json.load(open(fn)))\n",
    "\n",
    "    nb_annotated_tweets = compute_annotation_progress()\n",
    "    if label_column_name in df.keys():\n",
    "        print('Labels from', annotator_name, 'already in data!')\n",
    "        if compute_annotation_progress() < len(df):\n",
    "            print('Continuing annotation,', nb_annotated_tweets, 'of', len(df), 'already annotated')\n",
    "        else:\n",
    "            return\n",
    "    else:\n",
    "        df[label_column_name] = np.nan\n",
    "\n",
    "    print(help_text)\n",
    "    print('Starting annotation for', len(df) - nb_annotated_tweets, 'tweets:')\n",
    "    for index, row in df.iterrows():\n",
    "        if not pd.isna(row[label_column_name]):\n",
    "            continue\n",
    "        print(row.text)\n",
    "        label = process_input(input('\\t'))\n",
    "        if label is not None:\n",
    "            df.loc[index, label_column_name] = label\n",
    "        df.to_json(annotated_df_fn)\n",
    "\n",
    "    print('Finished!\\nSaved results as', annotated_df_fn, '\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Annotation Task\n",
    "\n",
    "Please refere to the annotation guide file for annotation examples. In case something is not clear feel free to ask.  \n",
    "\n",
    "IMPORTANT:   \n",
    "- For **F-Score**, we are only interested in the presence or absence of a location within the tweet. The context in which the location is mentioned is not important\n",
    "\n",
    "The label is either `1` if the tweet has a location or `0` otherwise.  \n",
    "\n",
    "For more information see the annotation guide.\n",
    "\n",
    "#### To start annotating run the cell below.  \n",
    "Press q to pause the annotation (the red error is intended bahviour).  \n",
    "Press p to show your progress.  \n",
    "Press h to see all possible functions."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-11-19T14:18:27.526842Z",
     "start_time": "2018-11-19T14:18:22.934442Z"
    },
    "hidden": true,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ss_california_tweets_annotatedHelen_Mudiwa_fscore.json already exists, continuing previous annotation process\n",
      "Labels from Helen_Mudiwa_fscore already in data!\n",
      "Continuing annotation, 16 of 1902 already annotated\n",
      "Possible Commands\n",
      "('0', '1')\tpossible labels\n",
      "h\tshow this help\n",
      "p\tshow progress\n",
      "q\tquit\n",
      "\n",
      "Starting annotation for 1886 tweets:\n",
      "Right ankle hurts\n"
     ]
    }
   ],
   "source": [
    "list(map(annotate_tweet_df, dataset_file_names))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>text</th>\n",
       "      <th>place</th>\n",
       "      <th>long</th>\n",
       "      <th>lat</th>\n",
       "      <th>clean_text</th>\n",
       "      <th>GPE</th>\n",
       "      <th>FAC</th>\n",
       "      <th>ORG</th>\n",
       "      <th>LOC</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>I'm at My Home Gym in Pacifica, CA https://t....</td>\n",
       "      <td>Pacifica; CA</td>\n",
       "      <td>-122.500464</td>\n",
       "      <td>37.593650</td>\n",
       "      <td>Im at My Home Gym in Pacifica, CA</td>\n",
       "      <td>Pacifica, CA</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>_styledbym.e killed it with this #shadowroot #...</td>\n",
       "      <td>None</td>\n",
       "      <td>-121.989751</td>\n",
       "      <td>38.355840</td>\n",
       "      <td>styledbym.e killed it with this shadowroot col...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>Primigi Classic loafers for your boy or girl. ...</td>\n",
       "      <td>East Oakdale, CA</td>\n",
       "      <td>-120.829960</td>\n",
       "      <td>37.774330</td>\n",
       "      <td>Primigi Classic loafers for your boy or girl. ...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Primigi</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>Warriors single game tickets go on sale at 10...</td>\n",
       "      <td>San Jose; CA</td>\n",
       "      <td>-121.891766</td>\n",
       "      <td>37.332484</td>\n",
       "      <td>Warriors single game tickets go on sale at 10...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>I'm at Hardly Strictly Bluegrass in San Franc...</td>\n",
       "      <td>San Francisco; CA</td>\n",
       "      <td>-122.489542</td>\n",
       "      <td>37.771727</td>\n",
       "      <td>Im at Hardly Strictly Bluegrass in San Franci...</td>\n",
       "      <td>San Francisco, CA</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1897</th>\n",
       "      <td>1995</td>\n",
       "      <td>how do i change the font on my phone 😐</td>\n",
       "      <td>6fafb06c49df870f</td>\n",
       "      <td>-122.264870</td>\n",
       "      <td>37.998157</td>\n",
       "      <td>how do i change the font on my phone</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1898</th>\n",
       "      <td>1996</td>\n",
       "      <td>@Vanessaah_x I just ran out lol</td>\n",
       "      <td>ee2cfc9feb061a08</td>\n",
       "      <td>-120.845655</td>\n",
       "      <td>37.504062</td>\n",
       "      <td>at Vanessaahx I just ran out lol</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1899</th>\n",
       "      <td>1997</td>\n",
       "      <td>@muhbellsaywhaat loves me. I know she do.</td>\n",
       "      <td>8004d2bebcc13e8c</td>\n",
       "      <td>-122.039433</td>\n",
       "      <td>37.971872</td>\n",
       "      <td>at muhbellsaywhaat loves me. I know she do.</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1900</th>\n",
       "      <td>1998</td>\n",
       "      <td>#ElvistheCorgi #SamtheCorgi #GigitheFrenchie #...</td>\n",
       "      <td>None</td>\n",
       "      <td>-122.894090</td>\n",
       "      <td>38.479099</td>\n",
       "      <td>ElvistheCorgi SamtheCorgi GigitheFrenchie dog ...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1901</th>\n",
       "      <td>1999</td>\n",
       "      <td>Just posted a photo @ Bay Bridge, San Francis...</td>\n",
       "      <td>San Francisco; CA</td>\n",
       "      <td>-122.405494</td>\n",
       "      <td>37.769230</td>\n",
       "      <td>Just posted a photo at  Bay Bridge, San Franc...</td>\n",
       "      <td>San Francisco</td>\n",
       "      <td>Bay Bridge</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1902 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      Unnamed: 0                                               text  \\\n",
       "0              0   I'm at My Home Gym in Pacifica, CA https://t....   \n",
       "1              1  _styledbym.e killed it with this #shadowroot #...   \n",
       "2              2  Primigi Classic loafers for your boy or girl. ...   \n",
       "3              3   Warriors single game tickets go on sale at 10...   \n",
       "4              4   I'm at Hardly Strictly Bluegrass in San Franc...   \n",
       "...          ...                                                ...   \n",
       "1897        1995             how do i change the font on my phone 😐   \n",
       "1898        1996                    @Vanessaah_x I just ran out lol   \n",
       "1899        1997          @muhbellsaywhaat loves me. I know she do.   \n",
       "1900        1998  #ElvistheCorgi #SamtheCorgi #GigitheFrenchie #...   \n",
       "1901        1999   Just posted a photo @ Bay Bridge, San Francis...   \n",
       "\n",
       "                   place        long        lat  \\\n",
       "0           Pacifica; CA -122.500464  37.593650   \n",
       "1                   None -121.989751  38.355840   \n",
       "2       East Oakdale, CA -120.829960  37.774330   \n",
       "3           San Jose; CA -121.891766  37.332484   \n",
       "4      San Francisco; CA -122.489542  37.771727   \n",
       "...                  ...         ...        ...   \n",
       "1897    6fafb06c49df870f -122.264870  37.998157   \n",
       "1898    ee2cfc9feb061a08 -120.845655  37.504062   \n",
       "1899    8004d2bebcc13e8c -122.039433  37.971872   \n",
       "1900                None -122.894090  38.479099   \n",
       "1901   San Francisco; CA -122.405494  37.769230   \n",
       "\n",
       "                                             clean_text                GPE  \\\n",
       "0                     Im at My Home Gym in Pacifica, CA       Pacifica, CA   \n",
       "1     styledbym.e killed it with this shadowroot col...               None   \n",
       "2     Primigi Classic loafers for your boy or girl. ...               None   \n",
       "3      Warriors single game tickets go on sale at 10...               None   \n",
       "4      Im at Hardly Strictly Bluegrass in San Franci...  San Francisco, CA   \n",
       "...                                                 ...                ...   \n",
       "1897              how do i change the font on my phone                None   \n",
       "1898                   at Vanessaahx I just ran out lol               None   \n",
       "1899        at muhbellsaywhaat loves me. I know she do.               None   \n",
       "1900  ElvistheCorgi SamtheCorgi GigitheFrenchie dog ...               None   \n",
       "1901   Just posted a photo at  Bay Bridge, San Franc...      San Francisco   \n",
       "\n",
       "             FAC      ORG   LOC  \n",
       "0           None     None  None  \n",
       "1           None     None  None  \n",
       "2           None  Primigi  None  \n",
       "3           None     None  None  \n",
       "4           None     None  None  \n",
       "...          ...      ...   ...  \n",
       "1897        None     None  None  \n",
       "1898        None     None  None  \n",
       "1899        None     None  None  \n",
       "1900        None     None  None  \n",
       "1901  Bay Bridge     None  None  \n",
       "\n",
       "[1902 rows x 10 columns]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df2 = pd.read_json('pd.read_json('ss_california_tweetsHelen_FS.json')')\n",
    "df2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {
    "height": "195px",
    "width": "282px"
   },
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}