Cleanup

Author: nottombrown

.gitignore

@@ -109,4 +109,5 @@ venv.bak/
 /site
 
 # mypy
-.mypy_cache/
+.mypy_cache/cloze_data
+cloze_data/

README.md

@@ -49,6 +49,7 @@ The ROCStories dataset can be downloaded from the associated [website](http://cs
 As with the [TensorFlow code](https://github.com/openai/finetune-transformer-lm), this code implements the ROCStories Cloze Test result reported in the paper which can be reproduced by running:
 
 ```bash
+python -m spacy download en
 python train.py --dataset rocstories --desc rocstories --submit --analysis --data_dir [path to data here]
 ```
 

model_py.py

@@ -1,14 +1,15 @@
-import re
-import math
-import json
 import copy
-import numpy as np
+import json
+import math
+import re
 
+import numpy as np
 import torch
 import torch.nn as nn
 import torch.nn.functional as F
 from torch.nn.parameter import Parameter
 
+
 def gelu(x):
     return 0.5*x*(1+torch.tanh(math.sqrt(2/math.pi)*(x+0.044715*torch.pow(x, 3))))
 
@@ -63,7 +64,7 @@ def forward(self, x):
 class Attention(nn.Module):
     def __init__(self, nx, n_ctx, cfg, scale=False):
         super(Attention, self).__init__()
-        n_state = nx # in Attention: n_state=768 (nx=n_embd) 
+        n_state = nx # in Attention: n_state=768 (nx=n_embd)
         #[switch nx => n_state from Block to Attention to keep identical to TF implem]
         assert n_state % cfg.n_head==0
         self.register_buffer('b', torch.tril(torch.ones(n_ctx, n_ctx)).view(1, 1, n_ctx, n_ctx))

setup.py

@@ -0,0 +1,25 @@
+import sys
+
+# py_version = (sys.version_info.major, sys.version_info.minor)
+# if py_version < (3, 6):
+#   raise ValueError(
+#     "This module is only compatible with Python 3.6+, but you are running "
+#     "Python {}. We recommend installing conda and adding it to your PATH:"
+#     "https://conda.io/docs/user-guide/install/index.html".format(py_version))
+
+from setuptools import setup
+
+
+setup(
+  name='lm',
+  packages=['lm'],
+  version='0.0.1',
+  install_requires=[
+    "ipdb",
+    'ftfy',
+    'spacy',
+    'pytorch',
+  ],
+  author='Tom B Brown',
+  author_email='[email protected]',
+)