first commit

config.py

@@ -0,0 +1,111 @@
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+from easydict import EasyDict as edict
+
+__C = edict()
+cfg = __C
+
+__C.gpu_id = 0
+__C.num_workers = 4
+__C.multi_gpus = False
+__C.seed = 666
+# training options
+__C.train = edict()
+__C.train.restore = False
+__C.train.lr = 0.0001
+__C.train.batch_size = 32
+__C.train.max_epochs = 25
+__C.train.vision_dim = 2048
+__C.train.word_dim = 300
+__C.train.module_dim = 512
+__C.train.train_num = 0 # Default 0 for full train set
+__C.train.restore = False
+__C.train.glove = True
+__C.train.k_max_frame_level = 16
+__C.train.k_max_clip_level = 8
+__C.train.spl_resolution = 1
+__C.train.caption_dim = 300
+__C.train = dict(__C.train)
+__C.train.pretrained_retrieve_path = None
+__C.train.joint = False
+__C.train.patch_number = 20000
+__C.train.topk = 3
+
+# validation
+__C.val = edict()
+__C.val.flag = True
+__C.val.val_num = 0 # Default 0 for full val set
+__C.val.topk = 10
+__C.val = dict(__C.val)
+# test
+__C.test = edict()
+__C.test.test_num = 0 # Default 0 for full test set
+__C.test.write_preds = False
+__C.test.visualization = False
+__C.test = dict(__C.test)
+# dataset options
+__C.dataset = edict()
+__C.dataset.name = 'tgif-qa' # ['tgif-qa', 'msrvtt-qa', 'msvd-qa']
+__C.dataset.question_type = 'none' #['frameqa', 'count', 'transition', 'action', 'none']
+__C.dataset.data_dir = ''
+__C.dataset.appearance_feat = '{}_{}_appearance_feat.h5'
+__C.dataset.motion_feat = '{}_{}_motion_feat.h5'
+__C.dataset.object_feat = '{}_{}_object_feat.h5'
+__C.dataset.vocab_json = '{}_{}_vocab.json'
+__C.dataset.train_question_pt = '{}_{}_train_questions.pt'
+__C.dataset.val_question_pt = '{}_{}_val_questions.pt'
+__C.dataset.test_question_pt = '{}_{}_test_questions.pt'
+__C.dataset.save_dir = ''
+__C.dataset.topk = 10
+__C.dataset.pretrained = ''
+__C.dataset.max_cap_num = 15
+__C.dataset = dict(__C.dataset)
+
+# experiment name
+__C.exp_name = 'defaultExp'
+
+# credit https://github.com/tohinz/pytorch-mac-network/blob/master/code/config.py
+def merge_cfg(yaml_cfg, cfg):
+    if type(yaml_cfg) is not edict:
+        return
+
+    for k, v in yaml_cfg.items():
+        if not k in cfg:
+            raise KeyError('{} is not a valid config key'.format(k))
+
+        old_type = type(cfg[k])
+        if old_type is not type(v):
+            if isinstance(cfg[k], np.ndarray):
+                v = np.array(v, dtype=cfg[k].dtype)
+            elif isinstance(cfg[k], list):
+                v = v.split(",")
+                v = [int(_v) for _v in v]
+            elif cfg[k] is None:
+                if v == "None":
+                    continue
+                else:
+                    v = v
+            else:
+                raise ValueError(('Type mismatch ({} vs. {}) '
+                                  'for config key: {}').format(type(cfg[k]),
+                                                               type(v), k))
+        # recursively merge dicts
+        if type(v) is edict:
+            try:
+                merge_cfg(yaml_cfg[k], cfg[k])
+            except:
+                print('Error under config key: {}'.format(k))
+                raise
+        else:
+            cfg[k] = v
+
+
+
+def cfg_from_file(file_name):
+    import yaml
+    with open(file_name, 'r') as f:
+        yaml_cfg = edict(yaml.load(f))
+
+    merge_cfg(yaml_cfg, __C)

configs/msrvtt_qa.yml

@@ -0,0 +1,37 @@
+gpu_id: 3
+multi_gpus: False
+num_workers: 8
+seed: 666
+exp_name: 'expMSRVTT-QA'
+
+train:
+  lr: 0.0001
+  batch_size: 16
+  restore: False
+  max_epochs: 25
+  word_dim: 300
+  module_dim: 512
+  glove: True
+  k_max_frame_level: 16
+  k_max_clip_level: 8
+  spl_resolution: 1
+  caption_dim : 768
+  joint : True
+  patch_number : 10000 # max ：50000
+  topk : 3
+
+val:
+  flag: True
+  topk : 3
+
+test:
+  test_num: 0
+  write_preds: False
+  visualization: False
+
+dataset:
+  name: 'msrvtt-qa'
+  question_type: 'none'
+  data_dir: 'data/msrvtt-qa'
+  save_dir: 'results/'
+  max_cap_num: 19   # max: 19

configs/msvd_qa.yml

@@ -0,0 +1,37 @@
+gpu_id: 0
+multi_gpus: False
+num_workers: 12
+seed: 666
+exp_name: 'expMSVD-QA'
+
+train:
+  lr: 0.0001
+  batch_size: 128
+  restore: False
+  max_epochs: 25
+  word_dim: 300
+  module_dim: 512
+  glove: True
+  k_max_frame_level: 16
+  k_max_clip_level: 8
+  spl_resolution: 1
+  caption_dim : 768
+  joint : True
+  patch_number : 5000  # max ：25000
+  topk : 3
+
+val:
+  flag: True
+  topk : 3
+
+test:
+  test_num: 0
+  write_preds: False
+  visualization: False
+
+dataset:
+  name: 'msvd-qa'
+  question_type: 'none'
+  data_dir: 'data/msvd-qa/'
+  save_dir: 'results/'
+  max_cap_num: 18 # max:18

configs/tgif_qa_action.yml

@@ -0,0 +1,34 @@
+gpu_id: 0
+multi_gpus: False
+num_workers: 8
+seed: 666
+exp_name: 'expTGIF-QAAction'
+
+train:
+  lr: 0.0001
+  batch_size: 64
+  restore: False
+  max_epochs: 30
+  word_dim: 300
+  module_dim: 512
+  glove: True
+  k_max_frame_level: 16
+  k_max_clip_level: 8
+  spl_resolution: 1
+  caption_dim : 300
+  joint : True
+  patch_number : 25000  # max ：25000
+  topk : 3
+val:
+  flag: True
+  topk : 5
+
+test:
+  test_num: 0
+  write_preds: False
+
+dataset:
+  name: 'tgif-qa'
+  question_type: 'action'
+  data_dir: 'data/tgif-qa/action'
+  save_dir: 'results/'

configs/tgif_qa_count.yml

@@ -0,0 +1,34 @@
+gpu_id: 0
+multi_gpus: False
+num_workers: 8
+seed: 666
+exp_name: 'expTGIF-QACount'
+
+train:
+  lr: 0.0001
+  batch_size: 64
+  restore: False
+  max_epochs: 30
+  word_dim: 300
+  module_dim: 512
+  glove: True
+  k_max_frame_level: 16
+  k_max_clip_level: 8
+  spl_resolution: 1
+  caption_dim : 300
+  joint : True
+  patch_number : 25000  # max ：25000
+  topk : 3
+val:
+  flag: True
+  topk : 5
+
+test:
+  test_num: 0
+  write_preds: False
+
+dataset:
+  name: 'tgif-qa'
+  question_type: 'count'
+  data_dir: 'data/tgif-qa/count'
+  save_dir: 'results/'

configs/tgif_qa_frameqa.yml

@@ -0,0 +1,34 @@
+gpu_id: 0
+multi_gpus: False
+num_workers: 8
+seed: 666
+exp_name: 'expTGIF-QAFrameQA'
+
+train:
+  lr: 0.0001
+  batch_size: 64
+  restore: False
+  max_epochs: 30
+  word_dim: 300
+  module_dim: 512
+  glove: True
+  k_max_frame_level: 16
+  k_max_clip_level: 8
+  spl_resolution: 1
+  caption_dim : 300
+  joint : True
+  patch_number : 25000  # max ：25000
+  topk : 3
+
+val:
+  flag: True
+  topk : 5
+test:
+  test_num: 0
+  write_preds: False
+
+dataset:
+  name: 'tgif-qa'
+  question_type: 'frameqa'
+  data_dir: 'data/tgif-qa/frameqa'
+  save_dir: 'results/'

configs/tgif_qa_transition.yml

@@ -0,0 +1,33 @@
+gpu_id: 0
+multi_gpus: False
+num_workers: 8
+seed: 666
+exp_name: 'expTGIF-QATransition'
+
+train:
+  lr: 0.0001
+  batch_size: 64
+  restore: False
+  max_epochs: 30
+  word_dim: 300
+  module_dim: 512
+  glove: True
+  k_max_frame_level: 16
+  k_max_clip_level: 8
+  spl_resolution: 1
+  caption_dim : 300
+  joint : True
+  patch_number : 25000  # max ：25000
+  topk : 3
+val:
+  flag: True
+  topk : 3
+test:
+  test_num: 0
+  write_preds: False
+
+dataset:
+  name: 'tgif-qa'
+  question_type: 'transition'
+  data_dir: 'data/tgif-qa/transition'
+  save_dir: 'results/'