TextFeatures.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas\n",
    "import email\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "import html2text\n",
    "\n",
    "dataset = pandas.read_msgpack('./data/development.msg', encoding='latin-1')\n",
    "dataset['email'] = dataset['email'].apply(email.message_from_string)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "corpus = []\n",
    "for (index, row) in dataset.iterrows():\n",
    "    if row['class'] == 'spam':\n",
    "        for part in row['email'].walk():\n",
    "            if part.get_content_type() == 'text/plain':\n",
    "                body = part.get_payload()\n",
    "            elif part.get_content_type() == 'text/html':\n",
    "                body = html2text.html2text(part.get_payload())\n",
    "        \n",
    "        corpus.append(body)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "MemoryError",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mMemoryError\u001b[0m                               Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-4-e950e1df90b3>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0mpca\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mRandomizedPCA\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mn_components\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m50\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mwhiten\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m \u001b[0mpca_model\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpca\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit_transform\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mwords_model\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtoarray\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32mc:\\python35\\lib\\site-packages\\scipy\\sparse\\compressed.py\u001b[0m in \u001b[0;36mtoarray\u001b[0;34m(self, order, out)\u001b[0m\n\u001b[1;32m    918\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mtoarray\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0morder\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mout\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m    919\u001b[0m         \u001b[1;34m\"\"\"See the docstring for `spmatrix.toarray`.\"\"\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m--> 920\u001b[0;31m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtocoo\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcopy\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mFalse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtoarray\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0morder\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0morder\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mout\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mout\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    921\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m    922\u001b[0m     \u001b[1;31m##############################################################\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mc:\\python35\\lib\\site-packages\\scipy\\sparse\\coo.py\u001b[0m in \u001b[0;36mtoarray\u001b[0;34m(self, order, out)\u001b[0m\n\u001b[1;32m    250\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mtoarray\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0morder\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mout\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m    251\u001b[0m         \u001b[1;34m\"\"\"See the docstring for `spmatrix.toarray`.\"\"\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m--> 252\u001b[0;31m         \u001b[0mB\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_process_toarray_args\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0morder\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mout\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    253\u001b[0m         \u001b[0mfortran\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mB\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mflags\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mf_contiguous\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m    254\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mfortran\u001b[0m \u001b[1;32mand\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mB\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mflags\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mc_contiguous\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mc:\\python35\\lib\\site-packages\\scipy\\sparse\\base.py\u001b[0m in \u001b[0;36m_process_toarray_args\u001b[0;34m(self, order, out)\u001b[0m\n\u001b[1;32m   1007\u001b[0m             \u001b[1;32mreturn\u001b[0m \u001b[0mout\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m   1008\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1009\u001b[0;31m             \u001b[1;32mreturn\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mzeros\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdtype\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0morder\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0morder\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1010\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m   1011\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m__numpy_ufunc__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfunc\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmethod\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mpos\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mMemoryError\u001b[0m: "
     ]
    }
   ],
   "source": [
    "import io\n",
    "\n",
    "vectorizer = CountVectorizer(min_df=1)\n",
    "words_model = vectorizer.fit_transform(corpus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from sklearn.decomposition import TruncatedSVD\n",
    "pca = TruncatedSVD(n_components=200)\n",
    "pca_model = pca.fit_transform(words_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "       ..., \n",
       "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "       [ 0.,  0.,  0., ...,  0.,  0.,  0.]])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pca.transform(vectorizer.transform(corpus[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "36091\n",
      "200\n"
     ]
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.base import BaseEstimator, TransformerMixin\n",
    "from sklearn.pipeline import FeatureUnion\n",
    "from sklearn.pipeline import Pipeline\n",
    "\n",
    "\n",
    "\n",
    "class FeatureTransformer(BaseEstimator, TransformerMixin):\n",
    "    def fit(self, x, y=None):\n",
    "        return self\n",
    "    \n",
    "    def generate(x):\n",
    "        return x\n",
    "    \n",
    "    def transform(self, data):\n",
    "        features = []\n",
    "        \n",
    "        for entry in data:\n",
    "            features.append(self.generate(data))\n",
    "        \n",
    "        return features\n",
    "    \n",
    "# Converts textual emails to python email objects\n",
    "class EmailGenerator(BaseEstimator, TransformerMixin):\n",
    "    def fit(self, x, y=None):\n",
    "        return self\n",
    "    \n",
    "    def transform(self, data):\n",
    "        import email\n",
    "        data['email'] = data['email'].apply(email.message_from_string)\n",
    "        return data\n",
    "\n",
    "class ContentTypeGenerator(BaseEstimator, TransformerMixin):\n",
    "    def __init__(self):\n",
    "        pass\n",
    "    \n",
    "    def fit(self, x, y=None):\n",
    "        return self\n",
    "    \n",
    "    def transform(self, data):\n",
    "        check = ['x-world', 'application', 'text', 'text/plain', 'text/html', 'video', 'audio', 'image', 'drawing', 'model', 'multipart', 'x-conference', 'i-world', 'music', 'message', 'x-music', 'www', 'chemical', 'paleovu', 'windows', 'xgl']\n",
    "        features = []\n",
    "        \n",
    "        for entry in data:\n",
    "            email = entry['email']\n",
    "            output = collections.defaultdict(bool)\n",
    "\n",
    "            for part in email.walk():\n",
    "                ct = part.get_content_type()\n",
    "\n",
    "                for kind in check:\n",
    "                    output['has_' + kind] |= ct.startswith(kind)\n",
    "            \n",
    "            features.append(output)\n",
    "        \n",
    "        return features\n",
    "    \n",
    "    \n",
    "class EmailCountsGenerator(BaseEstimator, TransformerMixin):\n",
    "    def __init__(self):\n",
    "        pass\n",
    "    \n",
    "    def fit(self, x, y=None):\n",
    "        return self\n",
    "    \n",
    "    def transform(self, data):\n",
    "        features = []\n",
    "        \n",
    "        for entry in data:\n",
    "            email = str(entry['email'])\n",
    "            \n",
    "            output = {\n",
    "                'length': len(email),\n",
    "                'spaces': email.count(' '),\n",
    "                'newlines': email.count('\\n')\n",
    "            }\n",
    "            \n",
    "            features.append(output)\n",
    "        \n",
    "        return features\n",
    "\n",
    "class ContentTypeGenerator(BaseEstimator, TransformerMixin):\n",
    "    def __init__(self):\n",
    "        pass\n",
    "    \n",
    "    def fit(self, x, y=None):\n",
    "        return self\n",
    "    \n",
    "    def transform(self, data):\n",
    "        features = []\n",
    "        \n",
    "        for entry in data:\n",
    "            email = entry['email']\n",
    "            output = {}\n",
    "            \n",
    "            features.append(output)\n",
    "        \n",
    "        return features\n",
    "    \n",
    "class FeatureGeneratorBase(BaseEstimator, TransformerMixin):\n",
    "    def __init__(self):\n",
    "        pass\n",
    "    \n",
    "    def fit(self, x, y=None):\n",
    "        return self\n",
    "    \n",
    "    def transform(self, data):\n",
    "        features = np.recarray(shape=(len(data),),\n",
    "            dtype=[('subject', object), ('body', object)])\n",
    "        \n",
    "        return features\n",
    "\n",
    "    \n",
    "pipeline = Pipeline([\n",
    "        ('transform_email', EmailGenerator()),\n",
    "        ('features', FeatureUnion=(\n",
    "                transformer_list=[\n",
    "                    ('content_type_features', ContentTypeGenerator())\n",
    "                ]\n",
    "            ))\n",
    "    ])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}