Re-organized shuffled files, all scripts work from wikinews.db file, added Dataset Creation Notebook tutorial

Author: Ubuntu

.gitignore

@@ -0,0 +1 @@
+*.pyc

Dataset SQLite3 Example.ipynb

@@ -0,0 +1,97 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# SQLite Creation"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Example of creating a dataset that will be compatible with [Summary Loop](https://github.com/CannyLab/summary_loop) training scripts.\n",
+    "\n",
+    "Another option is to modify the [collate_fn](https://pytorch.org/docs/stable/data.html) of the scripts, to interface with another data format.\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import sqlite3, os # This is a built in, no need to install\n",
+    "\n",
+    "example_dataset = [\n",
+    "    {\"title\": \"Example document1\", \"body\": \"The body of the very first document in the collection\"},\n",
+    "    {\"title\": \"Example document2\", \"body\": \"The body of the second document in the collection. You could put any data in here.\"},\n",
+    "]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "db_file = \"/home/phillab/dataset/example_dataset.db\"\n",
+    "\n",
+    "if os.path.isfile(db_file):\n",
+    "    os.remove(db_file)\n",
+    "\n",
+    "conn = sqlite3.connect(db_file,detect_types=sqlite3.PARSE_DECLTYPES)\n",
+    "conn.row_factory = sqlite3.Row\n",
+    "c = conn.cursor()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# CREATE TABLE\n",
+    "\n",
+    "sql_create = \"CREATE TABLE articles (id INTEGER PRIMARY KEY AUTOINCREMENT, title TEXT NOT NULL, body TEXT);\"\n",
+    "c.execute(sql_create)\n",
+    "conn.commit()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "sql_insert = \"INSERT INTO articles (title, body) VALUES (?, ?)\"\n",
+    "\n",
+    "for a in example_dataset:\n",
+    "    c.execute(sql_insert, (a['title'], a['body']))\n",
+    "conn.commit()"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.6.10"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

coverage.py renamed to model_coverage.py

@@ -2,8 +2,9 @@
 from pytorch_transformers.tokenization_bert import BertTokenizer
 from pytorch_transformers.modeling_bert import BertForPreTraining
 from torch.utils.data import DataLoader, RandomSampler
+import torch, os, time, utils_misc, argparse
+from utils_dataset import SQLDataset
 from utils_logplot import LogPlot
-import torch, os, time, utils_hdf5, argparse
 import random
 
 parser = argparse.ArgumentParser()
@@ -96,7 +97,7 @@ def convert_example_to_features(tokens_a, tokens_b, max_seq_length, tokenizer):
     return input_ids, input_mask, segment_ids, lm_label_ids
 
 def collate_func(inps):
-    bodies = [inp[0].decode() for inp in inps]
+    bodies = [inp['body'] for inp in inps]
     bodies_tokenized = [tokenizer.tokenize(body) for body in bodies]
 
     max_length = 400
@@ -131,7 +132,7 @@ def collate_func(inps):
 
     return batch_ids, batch_mask, batch_segments, batch_lm_label_ids, batch_is_next
 
-dataset = utils_hdf5.HDF5Dataset(args.dataset_file, collection_name="name")
+dataset = SQLDataset(args.dataset_file)
 dataloader = DataLoader(dataset=dataset, batch_size=2*args.train_batch_size, sampler=RandomSampler(dataset), drop_last=True, collate_fn=collate_func)
 
 param_optimizer = list(model.named_parameters())

fluency.py renamed to model_guardrails.py

@@ -1,33 +1,37 @@
-from transformers.optimization import AdamW, WarmupLinearSchedule
-import torch.utils.data
+from transformers.optimization import AdamW
 from torch.utils.data import DataLoader, RandomSampler
 
-import tqdm, nltk, torch, time, numpy as np
-import argparse, os
+import tqdm, nltk, torch, time, numpy as np, argparse, os
 from utils_logplot import LogPlot
-from coverage import KeywordCoverage
-import utils_hdf5 
+from model_coverage import KeywordCoverage
+from utils_dataset import SQLDataset
+import utils_misc 
 
 parser = argparse.ArgumentParser()
 parser.add_argument("--experiment", type=str, required=True, help="Experiment name. Will be used to save a model file and a log file.")
+parser.add_argument("--dataset_file", type=str, required=True, help="Which dataset file to use. Can be full path or the root folder will be attached.")
+
 parser.add_argument("--train_batch_size", type=int, default=8, help="Training batch size.")
 parser.add_argument("--n_kws", type=int, default=15, help="Top n words (tf-idf wise) will be masked in the coverage model.")
 parser.add_argument("--device", type=str, default="cuda", help="cuda or cpu")
 parser.add_argument('--fp16', action='store_true', help="Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit")
 
+models_folder = "/home/ubuntu/models/"
+logs_folder =   "/home/ubuntu/logs/"
+
+
 args = parser.parse_args()
 
 if args.device == "cuda":
-    freer_gpu = str(utils_hdf5.get_freer_gpu())
+    freer_gpu = str(utils_misc.get_freer_gpu())
     os.environ["CUDA_VISIBLE_DEVICES"] = ""+str(freer_gpu)
     args.experiment += "_"+freer_gpu
 
-def collate_func(inps):
-    return [inp[0].decode() for inp in inps], [inp[1].decode() for inp in inps]
+def collate_func(documents):
+    # When pretraining the coverage model, can feed real summaries, or the first K words of the document as summaries (for full unsupervised).
+    return [utils_misc.cut300(doc['body']) for doc in documents], [" ".join(doc['body'].split()[:50]) for doc in documents]
 
-models_folder = "/home/phillab/models/"
-# dataset = utils_hdf5.HDF5Dataset("/home/phillab/dataset/nl_quality_summaries.0.2.hdf5", collection_name="name")
-dataset = utils_hdf5.HDF5Dataset("/home/phillab/dataset/cnndm_training.hdf5", collection_name="name")
+dataset = SQLDataset(args.dataset_file)
 dataloader = DataLoader(dataset=dataset, batch_size=args.train_batch_size, sampler=RandomSampler(dataset), drop_last=True, collate_fn=collate_func)
 
 kw_cov = KeywordCoverage(args.device, keyword_model_file=os.path.join(models_folder, "keyword_extractor.joblib"), n_kws=args.n_kws) # , model_file=os.path.join(models_folder, "news_bert_bs64.bin")
@@ -42,8 +46,7 @@ def collate_func(inps):
 ]
 
 optimizer = AdamW(optimizer_grouped_parameters, lr=2e-5)
-scheduler = WarmupLinearSchedule(optimizer, warmup_steps=0, t_total=len(dataloader))
-logplot = LogPlot("/home/phillab/logs/coverage/bert_coverage_"+args.experiment+".log")
+logplot = LogPlot(os.path.join(logs_folder, "coverage/bert_coverage_%s.log" % (args.experiment)))
 
 if args.fp16:
     try:
@@ -65,12 +68,11 @@ def collate_func(inps):
         loss.backward()
 
     if ib%optim_every == 0:
-        scheduler.step()  # Update learning rate schedule
         optimizer.step()
         optimizer.zero_grad()
 
     logplot.cache({"loss": loss.item(), "accuracy": acc, "count": len(batch)}, prefix="T_")
     if time.time()-time_save > 60.0:
         logplot.save(printing=True)
         time_save = time.time()
-        kw_cov.save_model("/home/phillab/models/bert_coverage_"+args.experiment+".bin")
+        kw_cov.save_model(os.path.join(models_folder, "bert_coverage_%s.bin" % (args.experiment)))

pretrain_bert.py

@@ -1,23 +1,18 @@
 from transformers.optimization import AdamW
 from model_generator import GeneTransformer
 from torch.utils.data import DataLoader, RandomSampler
+import torch, os, time, argparse, tqdm
+from utils_dataset import SQLDataset
 from utils_logplot import LogPlot
-import torch, os, time, argparse
 from datetime import datetime
-import utils_hdf5
-import getpass, tqdm
-
-# user = os.getlogin()
-user = getpass.getuser()
+import utils_misc
 
 parser = argparse.ArgumentParser()
 parser.add_argument("--experiment", type=str, required=True, help="Experiment name. Will be used to save a model file and a log file.")
 parser.add_argument("--dataset_file", type=str, required=True, help="Which dataset file to use.")
-parser.add_argument("--task", type=str, help="Which generation task to perform. Can be: `cgen` (conditionally generate),  lm` (language modeling) or `copy`")
+parser.add_argument("--task", type=str, required=True, help="Which generation task to perform. Can be: `cgen` (conditionally generate),  lm` (language modeling) or `copy`. `cgen` is useful to train a supervised model, when data is available (for example a headline generator, summarizer, etc). `lm` is an unconditional language model, such as the GPT2 model, can be used to train a Fluency model. `copy` can be used to pretrain the generator for the summary_loop, this speeds up training of the summary_loop as the generator already starts with the strong baseline of copying the first K words of the input.")
 parser.add_argument("--max_output_length", required=True, type=int, help="Maximum output length. Saves time if the sequences are short.")
 
-parser.add_argument("--root_folder", type=str, default="/home/"+user+"/")
-parser.add_argument("--tokenizer", type=str, default="gpt2", help="Which tokenizer to use: gpt2 or bpecap.")
 parser.add_argument("--train_batch_size", type=int, default=8, help="Training batch size.")
 parser.add_argument("--n_epochs", type=int, default=3, help="Number of epochs to run over the data.")
 parser.add_argument("--optim_every", type=int, default=4, help="Optimize every x backprops. A multiplier to the true batch size.")
@@ -27,33 +22,27 @@
 
 args = parser.parse_args()
 
-models_folder = os.path.join(args.root_folder, "models/")
-logs_folder =   os.path.join(args.root_folder, "logs/")
+models_folder = "/home/ubuntu/models/"
+logs_folder =   "/home/ubuntu/logs/"
 
 if args.device == "cuda":
-    freer_gpu = str(utils_hdf5.get_freer_gpu())
+    freer_gpu = str(utils_misc.get_freer_gpu())
     os.environ["CUDA_VISIBLE_DEVICES"] = ""+str(freer_gpu)
     args.experiment += "_"+freer_gpu
 
 learning_rate = 2e-5
 n_epochs = args.n_epochs
 
-utils_hdf5.DoublePrint("printlog_generator_"+args.experiment+"_"+datetime.now().strftime("%Y-%m-%d")+".log", "a") ## << Wooh
-
-bpe_model = ""
-if args.tokenizer == "bpecap":
-    bpe_model = os.path.join(models_folder, "m.model")
-
-model = GeneTransformer(tokenizer_type=args.tokenizer, max_output_length=args.max_output_length, device=args.device, bpe_model=bpe_model)
+model = GeneTransformer(tokenizer_type="gpt2", max_output_length=args.max_output_length, device=args.device, bpe_model="")
 if len(args.starter_model) > 0:
     model.reload(os.path.join(models_folder, args.starter_model))
 
 print("Model loaded")
 
-def collate_func(inps):
-    return [inp[0] for inp in inps], [inp[1] for inp in inps]
+def collate_func(documents):
+    return [utils_misc.cut300(doc['body']) for doc in documents], [doc['title'] for doc in documents]
 
-dataset = utils_hdf5.HDF5Dataset(args.dataset_file, collection_name="name")
+dataset = SQLDataset(args.dataset_file)
 
 N = len(dataset)
 N_dev = 500
@@ -63,16 +52,14 @@ def collate_func(inps):
 dl_train = DataLoader(dataset=d_train, batch_size=args.train_batch_size, sampler=RandomSampler(d_train), collate_fn=collate_func)
 dl_dev   = DataLoader(dataset=d_dev,   batch_size=20, sampler=RandomSampler(d_dev), collate_fn=collate_func)
 
-# dataloader = DataLoader(dataset=dataset, batch_size=args.train_batch_size, sampler=RandomSampler(dataset), drop_last=True, collate_fn=collate_func)
-
 param_optimizer = list(model.model.named_parameters())
 no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
 optimizer_grouped_parameters = [
     {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.01},
     {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}
 ]
 
-logplot_file = os.path.join(logs_folder, "generator_"+args.experiment+".log")
+logplot_file = os.path.join(logs_folder, "generator_%s.log" % (args.experiment))
 summ = LogPlot(logplot_file)
 
 optimizer = AdamW(optimizer_grouped_parameters, lr=learning_rate)
@@ -89,14 +76,11 @@ def collate_func(inps):
 
 def map_batch(batch, task):
     sources, targets = batch
-    sources = [source.decode() for source in sources]
-    targets = [target.decode() for target in targets]
 
-    sources = [s for s in sources]
-    if task == "copy":
+    if task == "cgen":
+        pass # already in shape
+    elif task == "copy":
         targets = sources
-    elif task == "cgen":
-        targets = [t for t in targets]
     elif task == "lm":
         targets = sources
         sources = [""] * len(sources)

pretrain_coverage.py

@@ -1,15 +1,15 @@
+from torch.utils.data import DataLoader, RandomSampler
+import torch, os, sys, time, argparse, numpy as np
+from utils_dataset import SQLDataset, HDF5Dataset
 from transformers.optimization import AdamW
 from model_generator import GeneTransformer
-from torch.utils.data import DataLoader, RandomSampler
 from datetime import datetime, timedelta
 from utils_logplot import LogPlot
-import torch, os, sys, time, argparse, numpy as np
-import utils_hdf5, utils_tokenizer
+import utils_misc, utils_tokenizer
 
-from coverage import KeywordCoverage
-from fluency import PatternPenalty, LengthPenalty, RepeatPenalty
+from model_coverage import KeywordCoverage
+from model_guardrails import PatternPenalty, LengthPenalty, RepeatPenalty
 import threading, queue
-import torch.utils.data.dataset
 
 user = os.getlogin()
 
@@ -24,20 +24,20 @@
 parser.add_argument("--max_output_length", type=int, default=25, help="Maximum output length. Saves time if the sequences are short.")
 parser.add_argument("--save_every", type=int, default=60, help="Number of seconds between any two saves.")
 parser.add_argument("--device", type=str, default="cuda", help="cuda or cpu")
-parser.add_argument("--log_folder", type=str, default="", help="What should the model file start with.")
 parser.add_argument('--fp16', action='store_true', help="Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit")
 parser.add_argument("--ckpt_every", type=int, default=600, help="If 0, checkpointing is not used. Otherwise, checkpointing is done very x seconds.")
 parser.add_argument("--ckpt_lookback", type=int, default=300, help="When checkpointing, will consider the avg total score of the last x samples.")
 
 args = parser.parse_args()
 if args.device == "cuda":
-    freer_gpu = str(utils_hdf5.get_freer_gpu())
+    freer_gpu = str(utils_misc.get_freer_gpu())
     os.environ["CUDA_VISIBLE_DEVICES"] = ""+str(freer_gpu)
     args.experiment += "_"+freer_gpu
 
-models_folder = os.path.join(args.root_folder, "models/")
+models_folder = "/home/ubuntu/models/"
+log_folder = "/home/ubuntu/logs/"
+
 summarizer_model_start = os.path.join(models_folder, "gpt2_copier23.bin")
-args.log_folder = os.path.join(args.root_folder, "logs/", args.log_folder)
 
 ckpt_every = args.ckpt_every
 ckpt_lookback = int((args.ckpt_lookback+args.train_batch_size-1)/args.train_batch_size)
@@ -72,7 +72,7 @@ def collate_func(inps):
     {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}
 ]
 
-logplot_file = os.path.join(args.log_folder, "summary_loop_"+args.experiment+".log")
+logplot_file = os.path.join(log_folder, "summary_loop_%s.log" % (args.experiment))
 logplot = LogPlot(logplot_file)
 
 optimizer = AdamW(optimizer_grouped_parameters, lr=learning_rate)
@@ -106,9 +106,9 @@ def background_tokenizer(bodies, out_queue):
 print("Started training")
 
 if ".db" in args.dataset_file:
-    all_dataset = utils_hdf5.SQLDataset(args.dataset_file)
+    all_dataset = SQLDataset(args.dataset_file)
 else:
-    all_dataset = utils_hdf5.HDF5Dataset(args.dataset_file, collection_name="name")
+    all_dataset = HDF5Dataset(args.dataset_file, collection_name="name")
 
 dataset = all_dataset
 
@@ -234,7 +234,7 @@ def background_tokenizer(bodies, out_queue):
                 print("==============================================================================")
 
             if best_ckpt_score is None or current_score > best_ckpt_score:
-                print("[CKPT] Saved new best at:", current_score, "["+str(datetime.now())+"]")
+                print("[CKPT] Saved new best at: %.3f %s" % (current_score, "["+str(datetime.now())+"]"))
                 best_ckpt_score = current_score
                 torch.save(summarizer.model.state_dict(), ckpt_file)
                 torch.save(optimizer.state_dict(), ckpt_optimizer_file)