Initial commit

Author: c0stya

prep_corpus.py

@@ -0,0 +1,51 @@
+"""
+Preprocess dataset for sent140 corpus.
+
+"""
+import csv
+import numpy as np
+import cPickle as pickle
+import re
+import sys
+
+from utils import alph, translate, ALPH_rev
+
+FILENAME = './sent140.csv'
+LENGTH = 140
+
+def normalize(string):
+    '''Remove out-of-alphabet symbols and align to LENGTH '''
+    s = string.ljust(LENGTH)
+    return s
+
+def get_dataset(filename, limit=0):
+    X,Y = [], []
+    with open(filename) as h:
+        dataset = csv.reader(h)
+        i = 0
+        for row in dataset:
+            y = 0 if int(row[0]) == 0 else 1
+            Y.append(y)
+            x = translate(normalize(row[5][:140]))
+            X.append(x)
+            if limit !=0 and i > limit:
+                break
+            i+=1
+    return X,Y
+
+if __name__ == '__main__':
+
+    if len(sys.argv) < 3:
+        print "Usage: %s <input_corpus> <output>" % (sys.argv[0])
+        sys.exit(1)
+
+    inp = sys.argv[1]
+    out = sys.argv[2]
+
+    X,Y = get_dataset(filename=inp, limit=100000)
+
+    with open(out, 'wb') as h:
+        np.save(h, [len(alph)])
+        np.save(h, np.array(X, dtype=np.int8))
+        np.save(h, np.array(Y, dtype=np.int8))
+

prep_layer.py

@@ -0,0 +1,40 @@
+"""
+Preprocess dataset for sent140 corpus.
+
+"""
+import numpy as np
+import sys
+
+from utils import alph, translate, ALPH_rev
+
+def encode(data, alph):
+    m, n = data.shape # batch_size x n_steps
+
+    enc = np.zeros((m,n*alph), dtype=np.uint8)
+
+    for i in xrange(m):
+        for j in xrange(n):
+            enc[i, j*alph + data[i,j]] = 1
+
+    return enc
+
+if __name__ == '__main__':
+
+    if len(sys.argv) < 3:
+        print "Usage: %s <input_corpus> <output>" % (sys.argv[0])
+        sys.exit(1)
+
+    inp = sys.argv[1]
+    out = sys.argv[2]
+
+    with open(inp) as h:
+        alph = np.load(h)[0]
+        X = np.load(h)
+        Y = np.load(h)
+
+    # this could be memory-critical part
+    enc_X = encode(X, alph)
+
+    with open(out, 'wb') as h:
+        np.save(h, enc_X) # indicates the layer
+