Refactored and added comments for clarity

Author: Julian Bayardo

Analysis.ipynb

@@ -2623,6 +2623,71 @@
     "df"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": 32,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "0.973493614746 0.0201790309039\n"
+     ]
+    }
+   ],
+   "source": [
+    "import sklearn.tree\n",
+    "from sklearn.cross_validation import cross_val_score\n",
+    "from sklearn.metrics import confusion_matrix\n",
+    "import numpy as np\n",
+    "\n",
+    "dataset = df[[x for x in df.columns if x != 'class']].values\n",
+    "labels = df['class'].apply(lambda x: x == 1)\n",
+    "model = sklearn.tree.DecisionTreeClassifier()\n",
+    "res = cross_val_score(model, dataset, labels, cv=10, scoring='accuracy')\n",
+    "print(np.mean(res), np.std(res))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 31,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [],
+   "source": [
+    "\n",
+    "model = sklearn.tree.DecisionTreeClassifier()\n",
+    "model = model.fit(dataset, labels)\n",
+    "sklearn.tree.export_graphviz(model.tree_)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 33,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "array([ 0.99888951,  0.99930594,  0.99944475,  0.99458559,  0.96126076,\n",
+       "        0.9583449 ,  0.9566787 ,  0.95362399,  0.9555679 ,  0.9572341 ])"
+      ]
+     },
+     "execution_count": 33,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "res"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,

data_split.py

@@ -1,6 +1,6 @@
 import json
-import pandas as pd
-import numpy as np
+import pandas
+import numpy
 
 # Leo los mails (poner los paths correctos).
 ham_txt = json.load(open('./data/ham_dev.json'))
@@ -23,11 +23,11 @@
     output.append(current)
 
 # This is the merged, created dataset
-df = pd.DataFrame(output)
+df = pandas.DataFrame(output)
 
 # Split and save holdout and training data
 # We hold out about 10% of the data
-mask = np.random.rand(len(df)) < 0.8
+mask = numpy.random.rand(len(df)) < 0.8
 
 development = df[mask]
 development.to_msgpack('./data/development.msg')

load_development_dataset.py

@@ -103,9 +103,9 @@ def generate_upper_to_lower_case_ratios(email):
     return output
 
 def generate_subject_features(email):
-    def get_subject(x):
+    def get_subject(email):
         try:
-            s = re.search(r'^(fwd|re|fw):', x['subject'], re.IGNORECASE)
+            s = re.search(r'^(fwd|re|fw):', email['subject'], re.IGNORECASE)
 
             if s is not None:
                 return s.group(1).lower()
@@ -127,6 +127,7 @@ def get_subject(x):
     return output
 
 # Functions which create the output features
+# These must always return a dictionary with the same keys for eveyr row, all of which must be non-null
 transforms = [
     lambda email: {'length': len(email)},
     generate_content_types,
@@ -136,31 +137,35 @@ def get_subject(x):
     generate_upper_to_lower_case_ratios,
     generate_subject_features]
 
-# Set up thread pool
+# Process a single row, as received from the pandas.DataFrame iterator
 def transform_row(x):
     (index, row) = x
 
     current = {
         'class': row['class']
     }
 
+    # Apply the transform features to the email object
     for function in transforms:
         current.update(function(row['email']))
 
     return current
 
+# WARNING: This check is required to avoid loading the dataset again when creating a new thread
 if __name__ == '__main__':
     import multiprocessing
     import email
 
-    print("Loading data")
+    # Load dataset
     dataset = pandas.read_msgpack('./data/development.msg', encoding='latin-1')
     dataset['email'] = dataset['email'].apply(email.message_from_string)
 
-    print("Processing")
+    # Set up multiprocessing pool
+    # WARNING: number of threads should be no more than #cores + 1
     pool = multiprocessing.Pool(4)
+    # Generate features for every row
     transformed = pool.map(transform_row, dataset.iterrows())
 
-    print("Dumping to disk")
+    # Dump the processed dataset to disk
     preprocessed = pandas.DataFrame(transformed)
     preprocessed.to_msgpack('./data/processed.msg')

preprocessing.py

@@ -1,11 +1,16 @@
+import pandas
 from sklearn.tree import DecisionTreeClassifier
 from sklearn.cross_validation import cross_val_score
 from sklearn.metrics import confusion_matrix
-from transform_data_by_features import ds
 
-dataset = ds[[x for x in ds.columns if x != 'class']].values
-labels = ds['class'].apply(lambda x: x == 'spam')
+# Load processed data
+dataset = pandas.read_msgpack('./data/processed.msg', encoding='latin-1')
 
+# Separate features and labels
+features = dataset[[x for x in dataset.columns if x != 'class']].values
+labels = dataset['class'].apply(lambda x: x == 1)
+
+# Train model
 model = DecisionTreeClassifier()
-res = cross_val_score(model, dataset, labels, cv=10, scoring='roc_auc')
+res = cross_val_score(model, features, labels, cv=10, scoring='roc_auc')
 print(np.mean(res), np.std(res))