Initial branch

Author: rushter

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2016 Artem Golubin
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,14 @@
+# Machine learning algorithms 
+Minimal and clean examples of machine learning algorithms implemented in numpy. 
+
+
+
+Implemented: 
+* Deep learning (MLP, CNN, RNN, LSTM)
+* Linear regression, logistic regression
+* Random Forests
+* SVM with kernels
+* K-Means
+* PCA
+
+

examples/kmeans.py

@@ -0,0 +1,21 @@
+
+
+import numpy as np
+from sklearn.datasets import make_blobs
+
+from mla.kmeans import KMeans
+
+
+def kmeans_example(plot=False):
+    X, y = make_blobs(centers=4, n_samples=500, n_features=2, shuffle=True, random_state=42)
+    clusters = len(np.unique(y))
+    k = KMeans(K=clusters, max_iters=150, init='++')
+    k.fit(X)
+    k.predict()
+
+    if plot:
+        k.plot()
+
+
+if __name__ == '__main__':
+    kmeans_example(plot=True)

examples/linear_models.py

@@ -0,0 +1,38 @@
+
+import logging
+
+from sklearn.cross_validation import train_test_split
+from sklearn.datasets import make_classification
+from sklearn.datasets import make_regression
+
+from mla.linear_models import LinearRegression, LogisticRegression
+from mla.metrics.metrics import *
+
+# Change to DEBUG to see convergence
+logging.basicConfig(level=logging.ERROR)
+
+
+def regression():
+    X, y = make_regression(n_samples=10000, n_features=100, n_informative=75, n_targets=1,
+                           noise=0.05, random_state=1111, bias=0.5)
+
+    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25, random_state=1111)
+    model = LinearRegression(lr=0.01, max_iters=2000, penalty='l2', C=0.03)
+    model.fit(X_train, y_train)
+    predictions = model.predict(X_test)
+    print('regression mse', mean_squared_error(y_test, predictions))
+
+
+def classification():
+    X, y = make_classification(n_samples=1000, n_features=100, n_informative=75, random_state=1111, n_classes=2,
+                               class_sep=2.5, )
+    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=1111)
+    model = LogisticRegression(lr=0.01, max_iters=500, penalty='l1', C=0.01)
+    model.fit(X_train, y_train)
+    predictions = model.predict(X_test)
+    print('classification accuracy', accuracy(y_test, predictions))
+
+
+if __name__ == '__main__':
+    regression()
+    classification()

examples/nnet_convnet_mnist.py

@@ -0,0 +1,49 @@
+import logging
+
+from mla.datasets import load_mnist
+from mla.metrics import accuracy
+from mla.neuralnet import NeuralNet
+from mla.neuralnet.layers import Activation, Convolution, MaxPooling, Flatten, Dropout, Parameters
+from mla.neuralnet.layers import Dense
+from mla.neuralnet.optimizers import Adadelta
+from mla.utils import one_hot
+
+logging.basicConfig(level=logging.DEBUG)
+
+X_train, X_test, y_train, y_test = load_mnist()
+
+# Normalization
+X_train /= 255.
+X_test /= 255.
+
+y_train = one_hot(y_train.flatten())
+y_test = one_hot(y_test.flatten())
+print(X_train.shape, X_test.shape, y_train.shape, y_test.shape)
+
+# Approx. 15-20 min. per epoch
+model = NeuralNet(
+    layers=[
+        Convolution(n_filters=32, filter_shape=(3, 3), padding=(1, 1), stride=(1, 1)),
+        Activation('relu'),
+        Convolution(n_filters=32, filter_shape=(3, 3), padding=(1, 1), stride=(1, 1)),
+        Activation('relu'),
+        MaxPooling(pool_shape=(2, 2), stride=(2, 2)),
+        Dropout(0.5),
+
+        Flatten(),
+        Dense(128),
+        Activation('relu'),
+        Dropout(0.5),
+        Dense(10),
+        Activation('softmax'),
+    ],
+    loss='categorical_crossentropy',
+    optimizer=Adadelta(),
+    metric='accuracy',
+    batch_size=128,
+    max_epochs=3,
+)
+
+model.fit(X_train, y_train)
+predictions = model.predict(X_test)
+print accuracy(y_test, predictions)

examples/nnet_mlp.py

@@ -0,0 +1,76 @@
+
+import logging
+
+from sklearn.cross_validation import train_test_split
+from sklearn.datasets import make_classification
+from sklearn.datasets import make_regression
+from sklearn.metrics import roc_auc_score
+
+from mla.datasets import *
+from mla.metrics.metrics import root_mean_squared_log_error, mean_squared_error
+from mla.neuralnet import NeuralNet
+from mla.neuralnet.constraints import MaxNorm, UnitNorm
+from mla.neuralnet.layers import Activation, Dense, Dropout
+from mla.neuralnet.optimizers import SGD, RMSprop, Adagrad, Adadelta, Adam
+from mla.neuralnet.parameters import Parameters
+from mla.neuralnet.regularizers import *
+from mla.utils import one_hot
+
+logging.basicConfig(level=logging.DEBUG)
+
+
+def classification():
+    X, y = make_classification(n_samples=1000, n_features=100, n_informative=75, random_state=1111, n_classes=2,
+                               class_sep=2.5, )
+    y = one_hot(y)
+    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.15, random_state=1111)
+
+    model = NeuralNet(
+        layers=[
+            Dense(256, Parameters(init='uniform', regularizers={'W': L2(0.05)})),
+            Activation('relu'),
+            Dropout(0.5),
+            Dense(128, Parameters(init='normal', constraints={'W': MaxNorm()})),
+            Activation('relu'),
+            Dense(2),
+            Activation('softmax'),
+        ],
+        loss='categorical_crossentropy',
+        optimizer=Adadelta(),
+        metric='accuracy',
+        batch_size=64,
+        max_epochs=25,
+
+    )
+    model.fit(X_train, y_train)
+    predictions = model.predict(X_test)
+    print('classification accuracy', roc_auc_score(y_test[:, 0], predictions[:, 0]))
+
+
+def regression():
+    X, y = make_regression(n_samples=5000, n_features=25, n_informative=25, n_targets=1, random_state=100, noise=0.05)
+    y *= 0.01
+    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=1111)
+
+    model = NeuralNet(
+        layers=[
+            Dense(64, Parameters(init='normal')),
+            Activation('linear'),
+            Dense(32, Parameters(init='normal')),
+            Activation('linear'),
+            Dense(1),
+        ],
+        loss='mse',
+        optimizer=Adam(),
+        metric='mse',
+        batch_size=256,
+        max_epochs=15,
+    )
+    model.fit(X_train, y_train)
+    predictions = model.predict(X_test)
+    print("regression mse", mean_squared_error(y_test, predictions.flatten()))
+
+
+if __name__ == '__main__':
+    classification()
+    regression()

examples/nnet_rnn_binary_add.py

@@ -0,0 +1,68 @@
+import logging
+from itertools import combinations, islice
+
+import numpy as np
+from sklearn.cross_validation import train_test_split
+
+from mla.metrics import accuracy
+from mla.neuralnet import NeuralNet
+from mla.neuralnet.constraints import SmallNorm
+from mla.neuralnet.layers import Activation, TimeDistributedDense, Parameters
+from mla.neuralnet.layers.recurrent import RNN, LSTM
+from mla.neuralnet.optimizers import Adam
+
+logging.basicConfig(level=logging.DEBUG)
+
+
+def addition_dataset(dim=10, n_samples=10000, batch_size=64):
+    combs = list(islice(combinations(range(2 ** (dim - 1)), 2), n_samples))
+    binary_format = '{:0' + str(dim) + 'b}'
+    X = np.zeros((len(combs), dim, 2), dtype=np.uint8)
+    y = np.zeros((len(combs), dim, 1), dtype=np.uint8)
+
+    for i, (a, b) in enumerate(combs):
+        X[i, :, 0] = list(reversed([int(x) for x in binary_format.format(a)]))
+        X[i, :, 1] = list(reversed([int(x) for x in binary_format.format(b)]))
+        y[i, :, 0] = list(reversed([int(x) for x in binary_format.format(a + b)]))
+
+    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=1111)
+
+    train_b = (X_train.shape[0] // batch_size) * batch_size
+    test_b = (X_test.shape[0] // batch_size) * batch_size
+    X_train = X_train[0:train_b]
+    y_train = y_train[0:train_b]
+
+    X_test = X_test[0:test_b]
+    y_test = y_test[0:test_b]
+    return X_train, X_test, y_train, y_test
+
+
+def addition_nlp(ReccurentLayer):
+    X_train, X_test, y_train, y_test = addition_dataset(8, 5000)
+
+    print(X_train.shape, X_test.shape)
+    model = NeuralNet(
+        layers=[
+            ReccurentLayer,
+            TimeDistributedDense(1),
+            Activation('sigmoid'),
+        ],
+        loss='mse',
+        optimizer=Adam(),
+        metric='mse',
+        batch_size=64,
+        max_epochs=15,
+    )
+    # print X_train.shape
+    model.fit(X_train, y_train)
+    predictions = np.round(model.predict(X_test))
+    predictions = np.packbits(predictions.astype(np.uint8))
+    y_test = np.packbits(y_test.astype(np.int))
+    print(accuracy(y_test, predictions))
+
+
+
+# RNN
+# addition_nlp(RNN(16, parameters=Parameters(constraints={'W': SmallNorm(), 'U': SmallNorm()})))
+# LSTM
+addition_nlp(LSTM(16))

examples/nnet_rnn_text_generation.py

@@ -0,0 +1,80 @@
+from __future__ import print_function
+
+import logging
+import random
+
+import numpy as np
+import sys
+
+from mla.datasets import load_nietzsche
+from mla.neuralnet import NeuralNet
+from mla.neuralnet.constraints import SmallNorm
+from mla.neuralnet.layers import Activation, Dense, Parameters
+from mla.neuralnet.layers.recurrent import LSTM, RNN
+from mla.neuralnet.optimizers import Adam, RMSprop
+
+logging.basicConfig(level=logging.DEBUG)
+
+
+def sample(preds, temperature=1.0):
+    # helper function to sample an index from a probability array
+    preds = np.asarray(preds).astype('float64')
+    preds = np.log(preds) / temperature
+    exp_preds = np.exp(preds)
+    preds = exp_preds / np.sum(exp_preds)
+    probas = np.random.multinomial(1, preds, 1)
+    return np.argmax(probas)
+
+
+X, y, text, chars, char_indices, indices_char = load_nietzsche()
+# Round the number of sequences for batch processing
+items_count = X.shape[0] - (X.shape[0] % 64)
+maxlen = X.shape[1]
+X = X[0:items_count]
+y = y[0:items_count]
+
+print(X.shape, y.shape)
+# LSTM OR RNN
+# rnn_layer = RNN(128, return_sequences=False)
+rnn_layer = LSTM(128,return_sequences=False,)
+
+model = NeuralNet(
+    layers=[
+        rnn_layer,
+        # Flatten(),
+        # TimeStepSlicer(-1),
+        Dense(X.shape[2]),
+        Activation('softmax'),
+    ],
+    loss='categorical_crossentropy',
+    optimizer=RMSprop(learning_rate=0.01),
+    metric='accuracy',
+    batch_size=64,
+    max_epochs=1,
+    shuffle=False,
+
+)
+
+for i in xrange(25):
+    model.fit(X, y)
+    start_index = random.randint(0, len(text) - maxlen - 1)
+
+    generated = ''
+    sentence = text[start_index: start_index + maxlen]
+    generated += sentence
+    print('----- Generating with seed: "' + sentence + '"')
+    sys.stdout.write(generated)
+    for i in range(100):
+        x = np.zeros((64, maxlen, len(chars)))
+        for t, char in enumerate(sentence):
+            x[0, t, char_indices[char]] = 1.
+        preds = model.predict(x)[0]
+        next_index = sample(preds, 0.5)
+        next_char = indices_char[next_index]
+
+        generated += next_char
+        sentence = sentence[1:] + next_char
+
+        sys.stdout.write(next_char)
+        sys.stdout.flush()
+    print()

examples/pca.py

@@ -0,0 +1,23 @@
+from sklearn.cross_validation import train_test_split
+from sklearn.datasets import make_classification
+
+from mla.linear_models import LogisticRegression
+from mla.metrics import accuracy
+from mla.pca import PCA
+
+# logging.basicConfig(level=logging.DEBUG)
+
+X, y = make_classification(n_samples=1000, n_features=100, n_informative=75, random_state=1111, n_classes=2,
+                           class_sep=2.5, )
+
+for s in ['svd', 'eigen']:
+    p = PCA(15, solver=s)
+    p.fit(X)
+    X = p.transform(X)
+    print(X.shape)
+
+    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25, random_state=1111)
+    model = LogisticRegression(lr=0.001, max_iters=2500)
+    model.fit(X_train, y_train)
+    predictions = model.predict(X_test)
+    print('Classification accuracy for %s PCA: %s' % (s, accuracy(y_test, predictions)))