DarkVision dataset and configs added. ImageNetVid Data streaming code added that uses Zip files only

Author: khurramHashmi

configs/_base_/datasets/darkvision.py

@@ -0,0 +1,146 @@
+#dataset settings
+dataset_type = 'DarkVisionDataset'
+data_ann_root="../dataset_annotations/darkvision/high_illumination_3.2/annotations/"
+data_root="../dataset_annotations/darkvision/high_illumination_3.2/videos/"
+
+img_norm_cfg = dict(
+    mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True)
+train_pipeline = [
+    dict(type='LoadMultiImagesFromFile'),
+    dict(type='SeqLoadAnnotations', with_bbox=True, with_track=True),
+    dict(type='SeqResize', img_scale=(640, 640), keep_ratio=False),
+    dict(type='SeqRandomFlip', share_params=True, flip_ratio=0.5),
+    dict(type='SeqNormalize', **img_norm_cfg),
+    dict(type='SeqPad', size_divisor=32),
+    dict(
+        type='VideoCollect',
+        keys=['img', 'gt_bboxes', 'gt_labels', 'gt_instance_ids']),
+    dict(type='ConcatVideoReferences'),
+    dict(type='SeqDefaultFormatBundle', ref_prefix='ref')
+]
+test_pipeline = [
+    dict(type='LoadMultiImagesFromFile'),
+    dict(type='SeqResize', img_scale=(640, 640), keep_ratio=False),
+    dict(type='SeqRandomFlip', share_params=True, flip_ratio=0.0),
+    dict(type='SeqNormalize', **img_norm_cfg),
+    dict(type='SeqPad', size_divisor=32),
+    dict(
+        type='VideoCollect',
+        keys=['img'],
+        meta_keys=('num_left_ref_imgs', 'frame_stride')),
+    dict(type='ConcatVideoReferences'),
+    dict(type='MultiImagesToTensor', ref_prefix='ref'),
+    dict(type='ToList')
+]
+
+
+#Writing code to combine all jsons for training and test, repsectively
+#the code will only be executed once to generate list of path for train and test data splits
+# then it will be commented
+# import os
+# train_ann_files = os.listdir(os.path.join(data_ann_root, "train"))
+# train_img_files = os.listdir(os.path.join(data_root, "train"))
+#now same thing for test
+# train_ann_files = os.listdir(os.path.join(data_ann_root, "test"))
+# train_img_files = os.listdir(os.path.join(data_root, "test"))
+# #
+# for i in range(len(train_ann_files)):
+#     train_ann_files[i] = os.path.join(data_ann_root, "test", train_ann_files[i])
+#     train_img_files[i] = os.path.join(data_ann_root, "test", train_img_files[i], "tif-low-light")
+#
+# print(train_ann_files)
+# print("\n\n", train_img_files)
+#Writing code to combine all jsons for training and test, repsectively ENDS HERE
+
+
+train_ann_files=['../dataset_annotations/darkvision/high_illumination_3.2/annotations/train/0014.json',
+                '../dataset_annotations/darkvision/high_illumination_3.2/annotations/train/0019.json',
+                '../dataset_annotations/darkvision/high_illumination_3.2/annotations/train/0003.json', '../dataset_annotations/darkvision/high_illumination_3.2/annotations/train/0012.json', '../dataset_annotations/darkvision/high_illumination_3.2/annotations/train/0022.json', '../dataset_annotations/darkvision/high_illumination_3.2/annotations/train/0025.json', '../dataset_annotations/darkvision/high_illumination_3.2/annotations/train/0002.json', '../dataset_annotations/darkvision/high_illumination_3.2/annotations/train/0007.json', '../dataset_annotations/darkvision/high_illumination_3.2/annotations/train/0021.json', '../dataset_annotations/darkvision/high_illumination_3.2/annotations/train/0017.json', '../dataset_annotations/darkvision/high_illumination_3.2/annotations/train/0005.json', '../dataset_annotations/darkvision/high_illumination_3.2/annotations/train/0004.json', '../dataset_annotations/darkvision/high_illumination_3.2/annotations/train/0010.json', '../dataset_annotations/darkvision/high_illumination_3.2/annotations/train/0001.json', '../dataset_annotations/darkvision/high_illumination_3.2/annotations/train/0016.json', '../dataset_annotations/darkvision/high_illumination_3.2/annotations/train/0011.json', '../dataset_annotations/darkvision/high_illumination_3.2/annotations/train/0026.json', '../dataset_annotations/darkvision/high_illumination_3.2/annotations/train/0023.json', '../dataset_annotations/darkvision/high_illumination_3.2/annotations/train/0013.json', '../dataset_annotations/darkvision/high_illumination_3.2/annotations/train/0020.json', '../dataset_annotations/darkvision/high_illumination_3.2/annotations/train/0018.json', '../dataset_annotations/darkvision/high_illumination_3.2/annotations/train/0009.json', '../dataset_annotations/darkvision/high_illumination_3.2/annotations/train/0006.json', '../dataset_annotations/darkvision/high_illumination_3.2/annotations/train/0015.json', '../dataset_annotations/darkvision/high_illumination_3.2/annotations/train/0008.json', '../dataset_annotations/darkvision/high_illumination_3.2/annotations/train/0024.json']
+# print(train_ann_files)
+
+
+
+test_ann_files=['../dataset_annotations/darkvision/high_illumination_3.2/annotations/test/0027.json',
+                '../dataset_annotations/darkvision/high_illumination_3.2/annotations/test/0030.json',
+                '../dataset_annotations/darkvision/high_illumination_3.2/annotations/test/0031.json',
+                '../dataset_annotations/darkvision/high_illumination_3.2/annotations/test/0029.json',
+                '../dataset_annotations/darkvision/high_illumination_3.2/annotations/test/0032.json',
+                '../dataset_annotations/darkvision/high_illumination_3.2/annotations/test/0028.json']
+# test_ann_files=['../dataset_annotations/darkvision/high_illumination_3.2/annotations/test/0032.json']
+# test_img_files=['../dataset_annotations/darkvision/high_illumination_3.2/videos/test/0032/tif-low-light']
+train_img_files = ['../dataset_annotations/darkvision/high_illumination_3.2/videos/train/0011/tif-low-light',
+                   '../dataset_annotations/darkvision/high_illumination_3.2/videos/train/0007/tif-low-light',
+                   '../dataset_annotations/darkvision/high_illumination_3.2/videos/train/0010/tif-low-light',
+                   '../dataset_annotations/darkvision/high_illumination_3.2/videos/train/0009/tif-low-light',
+                   '../dataset_annotations/darkvision/high_illumination_3.2/videos/train/0013/tif-low-light',
+                   '../dataset_annotations/darkvision/high_illumination_3.2/videos/train/0015/tif-low-light',
+                   '../dataset_annotations/darkvision/high_illumination_3.2/videos/train/0001/tif-low-light',
+                   '../dataset_annotations/darkvision/high_illumination_3.2/videos/train/0018/tif-low-light',
+                   '../dataset_annotations/darkvision/high_illumination_3.2/videos/train/0017/tif-low-light',
+                   '../dataset_annotations/darkvision/high_illumination_3.2/videos/train/0004/tif-low-light',
+                   '../dataset_annotations/darkvision/high_illumination_3.2/videos/train/0005/tif-low-light',
+                   '../dataset_annotations/darkvision/high_illumination_3.2/videos/train/0002/tif-low-light',
+                   '../dataset_annotations/darkvision/high_illumination_3.2/videos/train/0023/tif-low-light',
+                   '../dataset_annotations/darkvision/high_illumination_3.2/videos/train/0026/tif-low-light',
+                   '../dataset_annotations/darkvision/high_illumination_3.2/videos/train/0021/tif-low-light',
+                   '../dataset_annotations/darkvision/high_illumination_3.2/videos/train/0014/tif-low-light',
+                   '../dataset_annotations/darkvision/high_illumination_3.2/videos/train/0019/tif-low-light',
+                   '../dataset_annotations/darkvision/high_illumination_3.2/videos/train/0006/tif-low-light',
+                   '../dataset_annotations/darkvision/high_illumination_3.2/videos/train/0022/tif-low-light',
+                   '../dataset_annotations/darkvision/high_illumination_3.2/videos/train/0008/tif-low-light',
+                   '../dataset_annotations/darkvision/high_illumination_3.2/videos/train/0024/tif-low-light',
+                   '../dataset_annotations/darkvision/high_illumination_3.2/videos/train/0003/tif-low-light',
+                   '../dataset_annotations/darkvision/high_illumination_3.2/videos/train/0012/tif-low-light',
+                   '../dataset_annotations/darkvision/high_illumination_3.2/videos/train/0025/tif-low-light',
+                   '../dataset_annotations/darkvision/high_illumination_3.2/videos/train/0020/tif-low-light',
+                   '../dataset_annotations/darkvision/high_illumination_3.2/videos/train/0016/tif-low-light']
+test_img_files=['../dataset_annotations/darkvision/high_illumination_3.2/videos/test/0028/tif-low-light',
+                '../dataset_annotations/darkvision/high_illumination_3.2/videos/test/0030/tif-low-light',
+                '../dataset_annotations/darkvision/high_illumination_3.2/videos/test/0031/tif-low-light',
+                '../dataset_annotations/darkvision/high_illumination_3.2/videos/test/0032/tif-low-light',
+                '../dataset_annotations/darkvision/high_illumination_3.2/videos/test/0027/tif-low-light',
+                '../dataset_annotations/darkvision/high_illumination_3.2/videos/test/0029/tif-low-light']
+
+train_ann_files.sort()
+train_img_files.sort()
+test_ann_files.sort()
+test_img_files.sort()
+data = dict(
+    samples_per_gpu=1,
+    workers_per_gpu=1,
+    train=dict(
+        type='DarkVisionDataset',
+        load_as_video=False,
+        ann_file= train_ann_files,
+        img_prefix=train_img_files,
+        ref_img_sampler=dict(
+            num_ref_imgs=2,
+            frame_range=9,
+            filter_key_img=True,
+            method='bilateral_uniform'),
+        pipeline=train_pipeline
+    ),
+    val=dict(
+        type='DarkVisionDataset',
+        load_as_video=False,
+        ann_file=test_ann_files,
+        img_prefix=test_img_files,
+        ref_img_sampler=dict(
+            num_ref_imgs=30,
+            frame_range=[-15, 15],
+            stride=1,
+            method='test_with_fix_stride'),
+        pipeline=test_pipeline,
+        test_mode=True),
+    test=dict(
+        type='DarkVisionDataset',
+        load_as_video=False,
+        ann_file=test_ann_files,
+        img_prefix=test_img_files,
+        ref_img_sampler=dict(
+            num_ref_imgs=30,
+            frame_range=[-15, 15],
+            stride=1,
+            method='test_with_fix_stride'),
+        pipeline=test_pipeline,
+        test_mode=True))

configs/_base_/datasets/imagenet_vid_fgfa_style.py

@@ -1,6 +1,9 @@
 # dataset settings
 dataset_type = 'ImagenetVIDDataset'
-data_root = 'data/ILSVRC/'
+# data_ann_root = '/ds-av/public_datasets/imagenet/pre/ILSVRC2015/COCO-Annotations/'
+data_ann_root = "../dataset_annotations/imagenetVID_2015/"
+data_root = "../dataset_annotations/imagenetVID_2015/Data/"
+# data_root = "/ds-av/public_datasets/imagenet/raw/Data/"
 img_norm_cfg = dict(
     mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True)
 train_pipeline = [
@@ -32,12 +35,12 @@
 ]
 data = dict(
     samples_per_gpu=1,
-    workers_per_gpu=2,
+    workers_per_gpu=1,
     train=[
         dict(
             type=dataset_type,
-            ann_file=data_root + 'annotations/imagenet_vid_train.json',
-            img_prefix=data_root + 'Data/VID',
+            ann_file=data_ann_root+'imagenet_vid_train.json',
+            img_prefix=data_root+'VID/',
             ref_img_sampler=dict(
                 num_ref_imgs=2,
                 frame_range=9,
@@ -47,8 +50,8 @@
         dict(
             type=dataset_type,
             load_as_video=False,
-            ann_file=data_root + 'annotations/imagenet_det_30plus1cls.json',
-            img_prefix=data_root + 'Data/DET',
+            ann_file=data_ann_root+'imagenet_det_30plus1cls.json',
+            img_prefix=data_root+'DET',
             ref_img_sampler=dict(
                 num_ref_imgs=2,
                 frame_range=0,
@@ -58,8 +61,8 @@
     ],
     val=dict(
         type=dataset_type,
-        ann_file=data_root + 'annotations/imagenet_vid_val.json',
-        img_prefix=data_root + 'Data/VID',
+        ann_file=data_ann_root+'imagenet_vid_val.json',
+        img_prefix=data_root+'VID',
         ref_img_sampler=dict(
             num_ref_imgs=30,
             frame_range=[-15, 15],
@@ -69,8 +72,8 @@
         test_mode=True),
     test=dict(
         type=dataset_type,
-        ann_file=data_root + 'annotations/imagenet_vid_val.json',
-        img_prefix=data_root + 'Data/VID',
+        ann_file=data_ann_root+'imagenet_vid_val.json',
+        img_prefix=data_root+'VID',
         ref_img_sampler=dict(
             num_ref_imgs=30,
             frame_range=[-15, 15],

configs/_base_/datasets/imagenet_vid_fgfa_style_datastreaming.py

@@ -0,0 +1,88 @@
+#This config is different from standard imagenet_vid_fgfa_style config in multiple ways:
+# - Instead of data directories, zip file paths are given for video frames
+# - LoadMultiImagesFromZipFile pipeline is used which is specifically created for streaming data in loading.py
+# - dataset_type is ImagenetVIDDatasetStreaming which is specifically created for streaming data in imagenet_vid_datastream.py
+
+
+# dataset settings
+dataset_type = 'ImagenetVIDDatasetStreaming'
+data_root = "/ds-av/public_datasets/imagenet/original/preprocessed/ILSVRC2015/"
+
+
+img_norm_cfg = dict(
+    mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True)
+train_pipeline = [
+    dict(type='LoadMultiImagesFromZipFile'),
+    dict(type='SeqLoadAnnotations', with_bbox=True, with_track=True),
+    dict(type='SeqResize', img_scale=(1000, 600), keep_ratio=True),
+    dict(type='SeqRandomFlip', share_params=True, flip_ratio=0.5),
+    dict(type='SeqNormalize', **img_norm_cfg),
+    dict(type='SeqPad', size_divisor=16),
+    dict(
+        type='VideoCollect',
+        keys=['img', 'gt_bboxes', 'gt_labels', 'gt_instance_ids']),
+    dict(type='ConcatVideoReferences'),
+    dict(type='SeqDefaultFormatBundle', ref_prefix='ref')
+]
+test_pipeline = [
+    dict(type='LoadMultiImagesFromZipFile'),
+    dict(type='SeqResize', img_scale=(1000, 600), keep_ratio=True),
+    dict(type='SeqRandomFlip', share_params=True, flip_ratio=0.0),
+    dict(type='SeqNormalize', **img_norm_cfg),
+    dict(type='SeqPad', size_divisor=16),
+    dict(
+        type='VideoCollect',
+        keys=['img'],
+        meta_keys=('num_left_ref_imgs', 'frame_stride')),
+    dict(type='ConcatVideoReferences'),
+    dict(type='MultiImagesToTensor', ref_prefix='ref'),
+    dict(type='ToList')
+]
+data = dict(
+    samples_per_gpu=1,
+    workers_per_gpu=1,
+    train=[
+        dict(
+            type=dataset_type,
+            ann_file=data_root+'COCO-Annotations/imagenet_vid_train.json',
+            img_prefix=data_root+'VID/train.zip',
+            ref_img_sampler=dict(
+                num_ref_imgs=2,
+                frame_range=9,
+                filter_key_img=True,
+                method='bilateral_uniform'),
+            pipeline=train_pipeline),
+        dict(
+            type=dataset_type,
+            load_as_video=False,
+            ann_file=data_root+'COCO-Annotations/imagenet_det_30plus1cls.json',
+            img_prefix=data_root+'DET/DET_train.zip',
+            ref_img_sampler=dict(
+                num_ref_imgs=2,
+                frame_range=0,
+                filter_key_img=False,
+                method='bilateral_uniform'),
+            pipeline=train_pipeline)
+    ],
+    val=dict(
+        type=dataset_type,
+        ann_file=data_root+'COCO-Annotations/imagenet_vid_val.json',
+        img_prefix=data_root+'VID/val.zip',
+        ref_img_sampler=dict(
+            num_ref_imgs=30,
+            frame_range=[-15, 15],
+            stride=1,
+            method='test_with_fix_stride'),
+        pipeline=test_pipeline,
+        test_mode=True),
+    test=dict(
+        type=dataset_type,
+        ann_file=data_root+'COCO-Annotations/imagenet_vid_val.json',
+        img_prefix=data_root+'VID/val.zip',
+        ref_img_sampler=dict(
+            num_ref_imgs=30,
+            frame_range=[-15, 15],
+            stride=1,
+            method='test_with_fix_stride'),
+        pipeline=test_pipeline,
+        test_mode=True))

configs/_base_/default_runtime.py

@@ -20,3 +20,5 @@
 opencv_num_threads = 0
 # set multi-process start method as `fork` to speed up the training
 mp_start_method = 'fork'
+
+auto_scale_lr = dict(enable=True, base_batch_size=16)

configs/_base_/models/deformable_detr_r50.py

@@ -0,0 +1,83 @@
+model = dict(
+    detector=dict(
+    type='DeformableDETR',
+    backbone=dict(
+        type='ResNet',
+        depth=50,
+        num_stages=4,
+        out_indices=(1, 2, 3),
+        frozen_stages=1,
+        norm_cfg=dict(type='BN', requires_grad=False),
+        norm_eval=True,
+        style='pytorch',
+        init_cfg=dict(type='Pretrained', checkpoint='torchvision://resnet50')),
+    neck=dict(
+        type='ChannelMapper',
+        in_channels=[512, 1024, 2048],
+        kernel_size=1,
+        out_channels=256,
+        act_cfg=None,
+        norm_cfg=dict(type='GN', num_groups=32),
+        num_outs=4),
+    bbox_head=dict(
+        type='DeformableDETRHead',
+        num_query=300,
+        num_classes=30,
+        in_channels=2048,
+        sync_cls_avg_factor=True,
+        as_two_stage=False,
+        transformer=dict(
+            type='DeformableDetrTransformer',
+            encoder=dict(
+                type='DetrTransformerEncoder',
+                num_layers=6,
+                transformerlayers=dict(
+                    type='BaseTransformerLayer',
+                    attn_cfgs=dict(
+                        type='MultiScaleDeformableAttention', embed_dims=256),
+                    feedforward_channels=1024,
+                    ffn_dropout=0.1,
+                    operation_order=('self_attn', 'norm', 'ffn', 'norm'))),
+            decoder=dict(
+                type='DeformableDetrTransformerDecoder',
+                num_layers=6,
+                return_intermediate=True,
+                transformerlayers=dict(
+                    type='DetrTransformerDecoderLayer',
+                    attn_cfgs=[
+                        dict(
+                            type='MultiheadAttention',
+                            embed_dims=256,
+                            num_heads=8,
+                            dropout=0.1),
+                        dict(
+                            type='MultiScaleDeformableAttention',
+                            embed_dims=256)
+                    ],
+                    feedforward_channels=1024,
+                    ffn_dropout=0.1,
+                    operation_order=('self_attn', 'norm', 'cross_attn', 'norm',
+                                     'ffn', 'norm')))),
+        positional_encoding=dict(
+            type='SinePositionalEncoding',
+            num_feats=128,
+            normalize=True,
+            offset=-0.5),
+        loss_cls=dict(
+            type='FocalLoss',
+            use_sigmoid=True,
+            gamma=2.0,
+            alpha=0.25,
+            loss_weight=2.0),
+        loss_bbox=dict(type='L1Loss', loss_weight=5.0),
+        loss_iou=dict(type='GIoULoss', loss_weight=2.0)),
+    # training and testing settings
+    train_cfg=dict(
+        assigner=dict(
+            type='HungarianAssigner',
+            cls_cost=dict(type='FocalLossCost', weight=2.0),
+            reg_cost=dict(type='BBoxL1Cost', weight=5.0, box_format='xywh'),
+            iou_cost=dict(type='IoUCost', iou_mode='giou', weight=2.0))),
+    test_cfg=dict(max_per_img=100)
+    )
+)