Add dataset loaders for news20, usps and mnist.

Author: mblondel

Makefile

@@ -0,0 +1,29 @@
+
+DATADIR=$(HOME)/lightning_data
+
+datadir:
+	mkdir -p $(DATADIR)
+
+download-news20: datadir
+	./download.sh http://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/multiclass/news20.scale.bz2
+	bunzip2 news20.scale.bz2
+	mv news20.scale $(DATADIR)
+	./download.sh http://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/multiclass/news20.t.scale.bz2
+	bunzip2 news20.t.scale.bz2
+	mv news20.t.scale $(DATADIR)
+
+download-usps: datadir
+	./download.sh http://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/multiclass/usps.bz2
+	bunzip2 usps.bz2
+	mv usps $(DATADIR)
+	./download.sh http://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/multiclass/usps.t.bz2
+	bunzip2 usps.t.bz2
+	mv usps.t $(DATADIR)
+
+download-mnist: datadir
+	./download.sh http://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/multiclass/mnist.scale.bz2
+	bunzip2 mnist.scale.bz2
+	mv mnist.scale $(DATADIR)
+	./download.sh http://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/multiclass/mnist.scale.t.bz2
+	bunzip2 mnist.scale.t.bz2
+	mv mnist.scale.t $(DATADIR)

download.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+# download a file with either wget or curl
+
+if [ "$1foo" = "foo" ]; then
+    echo "usage: `basename $0` url"
+    exit 1
+fi
+
+wget_path=`which wget`
+
+if [ $? = 0 ]; then
+    $wget_path $1
+    exit $?
+fi
+
+curl_path=`which curl`
+
+if [ $? = 0 ]; then
+    $curl_path -O $1
+    exit $?
+fi
+

lightning/datasets/__init__.py

@@ -0,0 +1 @@
+from .loaders import load_news20, load_usps, load_mnist

lightning/datasets/loaders.py

@@ -0,0 +1,36 @@
+import os
+
+try:
+    from svmlight_loader import load_svmlight_files
+except ImportError:
+    from sklearn.datasets import load_svmlight_files
+
+from sklearn.datasets.base import get_data_home as _get_data_home
+
+def get_data_home():
+    return _get_data_home().replace("scikit_learn", "lightning")
+
+def _load(train_file, test_file, name):
+    if not os.path.exists(train_file) or not os.path.exists(test_file):
+        raise IOError("Dataset missing! " +
+                      "Run 'make download-%s' at the project root." % name)
+
+    return load_svmlight_files((train_file, test_file))
+
+def load_news20():
+    data_home = get_data_home()
+    train_file = os.path.join(data_home, "news20.scale")
+    test_file = os.path.join(data_home, "news20.t.scale")
+    return _load(train_file, test_file, "news20")
+
+def load_usps():
+    data_home = get_data_home()
+    train_file = os.path.join(data_home, "usps")
+    test_file = os.path.join(data_home, "usps.t")
+    return _load(train_file, test_file, "usps")
+
+def load_mnist():
+    data_home = get_data_home()
+    train_file = os.path.join(data_home, "mnist.scale")
+    test_file = os.path.join(data_home, "mnist.scale.t")
+    return _load(train_file, test_file, "mnist")