Added argument --with_model, verbosity control, epoch elapsed time

Author: Paulo Querido

train.py

@@ -1,10 +1,15 @@
+import time
+start = time.perf_counter()
 import tensorflow as tf
 import argparse
 import pickle
 import os
 from model import Model
 from utils import build_dict, build_dataset, batch_iter
 
+# Uncomment next 2 lines to suppress error and Tensorflow info verbosity. Or change logging levels
+# tf.logging.set_verbosity(tf.logging.FATAL)
+# os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
 
 def add_arguments(parser):
     parser.add_argument("--num_hidden", type=int, default=150, help="Network size.")
@@ -20,6 +25,9 @@ def add_arguments(parser):
 
     parser.add_argument("--toy", action="store_true", help="Use only 50K samples of data")
 
+    parser.add_argument("--with_model", action="store_true", help="Continue from previously saved model")
+
+
 
 parser = argparse.ArgumentParser()
 add_arguments(parser)
@@ -29,6 +37,11 @@ def add_arguments(parser):
 
 if not os.path.exists("saved_model"):
     os.mkdir("saved_model")
+else:
+    if args.with_model:
+        old_model_checkpoint_path = open('saved_model/checkpoint', 'r')
+        old_model_checkpoint_path = "".join(["saved_model/",old_model_checkpoint_path.read().splitlines()[0].split('"')[1] ])
+
 
 print("Building dictionary...")
 word_dict, reversed_dict, article_max_len, summary_max_len = build_dict("train", args.toy)
@@ -40,11 +53,14 @@ def add_arguments(parser):
     model = Model(reversed_dict, article_max_len, summary_max_len, args)
     sess.run(tf.global_variables_initializer())
     saver = tf.train.Saver(tf.global_variables())
+    if 'old_model_checkpoint_path' in globals():
+        print("Continuing from previous trained model:" , old_model_checkpoint_path , "...")
+        saver.restore(sess, old_model_checkpoint_path )
 
     batches = batch_iter(train_x, train_y, args.batch_size, args.num_epochs)
     num_batches_per_epoch = (len(train_x) - 1) // args.batch_size + 1
 
-    print("Iteration starts.")
+    print("\nIteration starts.")
     print("Number of batches per epoch :", num_batches_per_epoch)
     for batch_x, batch_y in batches:
         batch_x_len = list(map(lambda x: len([y for y in x if y != 0]), batch_x))
@@ -72,5 +88,8 @@ def add_arguments(parser):
             print("step {0}: loss = {1}".format(step, loss))
 
         if step % num_batches_per_epoch == 0:
+            hours, rem = divmod(time.perf_counter() - start, 3600)
+            minutes, seconds = divmod(rem, 60)
             saver.save(sess, "./saved_model/model.ckpt", global_step=step)
-            print("Epoch {0}: Model is saved.".format(step // num_batches_per_epoch))
+            print(" Epoch {0}: Model is saved.".format(step // num_batches_per_epoch),
+            "Elapsed: {:0>2}:{:0>2}:{:05.2f}".format(int(hours),int(minutes),seconds) , "\n")