diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..1cfb227f --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +dat/yolo/yolov3-tiny.weights filter=lfs diff=lfs merge=lfs -text +dat/yolo/yolov3.weights filter=lfs diff=lfs merge=lfs -text diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 00000000..dd951827 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,18 @@ +cmake_minimum_required(VERSION 2.8 FATAL_ERROR) + +add_definitions(-std=c++11) + +set(CXX_FLAGS "-Wall") +set(CMAKE_CXX_FLAGS, "${CXX_FLAGS}") + +project(camera_fusion) + +find_package(OpenCV 4.1 REQUIRED) + +include_directories(${OpenCV_INCLUDE_DIRS}) +link_directories(${OpenCV_LIBRARY_DIRS}) +add_definitions(${OpenCV_DEFINITIONS}) + +# Executable for create matrix exercise +add_executable (3D_object_tracking src/camFusion_Student.cpp src/FinalProject_Camera.cpp src/lidarData.cpp src/matching2D_Student.cpp src/objectDetection2D.cpp) +target_link_libraries (3D_object_tracking ${OpenCV_LIBRARIES}) \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 00000000..e69de29b diff --git a/dat/yolo/coco.names b/dat/yolo/coco.names new file mode 100644 index 00000000..ca76c80b --- /dev/null +++ b/dat/yolo/coco.names @@ -0,0 +1,80 @@ +person +bicycle +car +motorbike +aeroplane +bus +train +truck +boat +traffic light +fire hydrant +stop sign +parking meter +bench +bird +cat +dog +horse +sheep +cow +elephant +bear +zebra +giraffe +backpack +umbrella +handbag +tie +suitcase +frisbee +skis +snowboard +sports ball +kite +baseball bat +baseball glove +skateboard +surfboard +tennis racket +bottle +wine glass +cup +fork +knife +spoon +bowl +banana +apple +sandwich +orange +broccoli +carrot +hot dog +pizza +donut +cake +chair +sofa +pottedplant +bed +diningtable +toilet +tvmonitor +laptop +mouse +remote +keyboard +cell phone +microwave +oven +toaster +sink +refrigerator +book +clock +vase +scissors +teddy bear +hair drier +toothbrush diff --git a/dat/yolo/yolov3-tiny.cfg b/dat/yolo/yolov3-tiny.cfg new file mode 100644 index 00000000..f46a4f12 --- /dev/null +++ b/dat/yolo/yolov3-tiny.cfg @@ -0,0 +1,182 @@ +[net] +# Testing +batch=1 +subdivisions=1 +# Training +# batch=64 +# subdivisions=2 +width=416 +height=416 +channels=3 +momentum=0.9 +decay=0.0005 +angle=0 +saturation = 1.5 +exposure = 1.5 +hue=.1 + +learning_rate=0.001 +burn_in=1000 +max_batches = 500200 +policy=steps +steps=400000,450000 +scales=.1,.1 + +[convolutional] +batch_normalize=1 +filters=16 +size=3 +stride=1 +pad=1 +activation=leaky + +[maxpool] +size=2 +stride=2 + +[convolutional] +batch_normalize=1 +filters=32 +size=3 +stride=1 +pad=1 +activation=leaky + +[maxpool] +size=2 +stride=2 + +[convolutional] +batch_normalize=1 +filters=64 +size=3 +stride=1 +pad=1 +activation=leaky + +[maxpool] +size=2 +stride=2 + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=1 +pad=1 +activation=leaky + +[maxpool] +size=2 +stride=2 + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=leaky + +[maxpool] +size=2 +stride=2 + +[convolutional] +batch_normalize=1 +filters=512 +size=3 +stride=1 +pad=1 +activation=leaky + +[maxpool] +size=2 +stride=1 + +[convolutional] +batch_normalize=1 +filters=1024 +size=3 +stride=1 +pad=1 +activation=leaky + +########### + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=512 +size=3 +stride=1 +pad=1 +activation=leaky + +[convolutional] +size=1 +stride=1 +pad=1 +filters=255 +activation=linear + + + +[yolo] +mask = 3,4,5 +anchors = 10,14, 23,27, 37,58, 81,82, 135,169, 344,319 +classes=80 +num=6 +jitter=.3 +ignore_thresh = .7 +truth_thresh = 1 +random=1 + +[route] +layers = -4 + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=leaky + +[upsample] +stride=2 + +[route] +layers = -1, 8 + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=leaky + +[convolutional] +size=1 +stride=1 +pad=1 +filters=255 +activation=linear + +[yolo] +mask = 0,1,2 +anchors = 10,14, 23,27, 37,58, 81,82, 135,169, 344,319 +classes=80 +num=6 +jitter=.3 +ignore_thresh = .7 +truth_thresh = 1 +random=1 \ No newline at end of file diff --git a/dat/yolo/yolov3-tiny.weights b/dat/yolo/yolov3-tiny.weights new file mode 100644 index 00000000..d357541a --- /dev/null +++ b/dat/yolo/yolov3-tiny.weights @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dccea06f59b781ec1234ddf8d1e94b9519a97f4245748a7d4db75d5b7080a42c +size 35434956 diff --git a/dat/yolo/yolov3.cfg b/dat/yolo/yolov3.cfg new file mode 100644 index 00000000..938ffff2 --- /dev/null +++ b/dat/yolo/yolov3.cfg @@ -0,0 +1,789 @@ +[net] +# Testing +# batch=1 +# subdivisions=1 +# Training +batch=64 +subdivisions=16 +width=608 +height=608 +channels=3 +momentum=0.9 +decay=0.0005 +angle=0 +saturation = 1.5 +exposure = 1.5 +hue=.1 + +learning_rate=0.001 +burn_in=1000 +max_batches = 500200 +policy=steps +steps=400000,450000 +scales=.1,.1 + +[convolutional] +batch_normalize=1 +filters=32 +size=3 +stride=1 +pad=1 +activation=leaky + +# Downsample + +[convolutional] +batch_normalize=1 +filters=64 +size=3 +stride=2 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=32 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=64 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + +# Downsample + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=2 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=64 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=64 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + +# Downsample + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=2 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + +# Downsample + +[convolutional] +batch_normalize=1 +filters=512 +size=3 +stride=2 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=512 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=512 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=512 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=512 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=512 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=512 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=512 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=512 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + +# Downsample + +[convolutional] +batch_normalize=1 +filters=1024 +size=3 +stride=2 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=1024 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=1024 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=1024 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=1024 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + +###################### + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=1024 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=1024 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=1024 +activation=leaky + +[convolutional] +size=1 +stride=1 +pad=1 +filters=255 +activation=linear + + +[yolo] +mask = 6,7,8 +anchors = 10,13, 16,30, 33,23, 30,61, 62,45, 59,119, 116,90, 156,198, 373,326 +classes=80 +num=9 +jitter=.3 +ignore_thresh = .7 +truth_thresh = 1 +random=1 + + +[route] +layers = -4 + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[upsample] +stride=2 + +[route] +layers = -1, 61 + + + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=512 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=512 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=512 +activation=leaky + +[convolutional] +size=1 +stride=1 +pad=1 +filters=255 +activation=linear + + +[yolo] +mask = 3,4,5 +anchors = 10,13, 16,30, 33,23, 30,61, 62,45, 59,119, 116,90, 156,198, 373,326 +classes=80 +num=9 +jitter=.3 +ignore_thresh = .7 +truth_thresh = 1 +random=1 + + + +[route] +layers = -4 + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=leaky + +[upsample] +stride=2 + +[route] +layers = -1, 36 + + + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=256 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=256 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=256 +activation=leaky + +[convolutional] +size=1 +stride=1 +pad=1 +filters=255 +activation=linear + + +[yolo] +mask = 0,1,2 +anchors = 10,13, 16,30, 33,23, 30,61, 62,45, 59,119, 116,90, 156,198, 373,326 +classes=80 +num=9 +jitter=.3 +ignore_thresh = .7 +truth_thresh = 1 +random=1 + diff --git a/dat/yolo/yolov3.weights b/dat/yolo/yolov3.weights new file mode 100644 index 00000000..550ca2f1 --- /dev/null +++ b/dat/yolo/yolov3.weights @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:523e4e69e1d015393a1b0a441cef1d9c7659e3eb2d7e15f793f060a21b32f297 +size 248007048 diff --git a/images/KITTI/2011_09_26/image_02/data/0000000000.png b/images/KITTI/2011_09_26/image_02/data/0000000000.png new file mode 100644 index 00000000..0d75986b Binary files /dev/null and b/images/KITTI/2011_09_26/image_02/data/0000000000.png differ diff --git a/images/KITTI/2011_09_26/image_02/data/0000000001.png b/images/KITTI/2011_09_26/image_02/data/0000000001.png new file mode 100644 index 00000000..c77dc22f Binary files /dev/null and b/images/KITTI/2011_09_26/image_02/data/0000000001.png differ diff --git a/images/KITTI/2011_09_26/image_02/data/0000000002.png b/images/KITTI/2011_09_26/image_02/data/0000000002.png new file mode 100644 index 00000000..88c6b0e2 Binary files /dev/null and b/images/KITTI/2011_09_26/image_02/data/0000000002.png differ diff --git a/images/KITTI/2011_09_26/image_02/data/0000000003.png b/images/KITTI/2011_09_26/image_02/data/0000000003.png new file mode 100644 index 00000000..146049bb Binary files /dev/null and b/images/KITTI/2011_09_26/image_02/data/0000000003.png differ diff --git a/images/KITTI/2011_09_26/image_02/data/0000000004.png b/images/KITTI/2011_09_26/image_02/data/0000000004.png new file mode 100644 index 00000000..968d0c02 Binary files /dev/null and b/images/KITTI/2011_09_26/image_02/data/0000000004.png differ diff --git a/images/KITTI/2011_09_26/image_02/data/0000000005.png b/images/KITTI/2011_09_26/image_02/data/0000000005.png new file mode 100644 index 00000000..e5201ab0 Binary files /dev/null and b/images/KITTI/2011_09_26/image_02/data/0000000005.png differ diff --git a/images/KITTI/2011_09_26/image_02/data/0000000006.png b/images/KITTI/2011_09_26/image_02/data/0000000006.png new file mode 100644 index 00000000..81483f99 Binary files /dev/null and b/images/KITTI/2011_09_26/image_02/data/0000000006.png differ diff --git a/images/KITTI/2011_09_26/image_02/data/0000000007.png b/images/KITTI/2011_09_26/image_02/data/0000000007.png new file mode 100644 index 00000000..c20f8707 Binary files /dev/null and b/images/KITTI/2011_09_26/image_02/data/0000000007.png differ diff --git a/images/KITTI/2011_09_26/image_02/data/0000000008.png b/images/KITTI/2011_09_26/image_02/data/0000000008.png new file mode 100644 index 00000000..d90354b7 Binary files /dev/null and b/images/KITTI/2011_09_26/image_02/data/0000000008.png differ diff --git a/images/KITTI/2011_09_26/image_02/data/0000000009.png b/images/KITTI/2011_09_26/image_02/data/0000000009.png new file mode 100644 index 00000000..96988732 Binary files /dev/null and b/images/KITTI/2011_09_26/image_02/data/0000000009.png differ diff --git a/images/KITTI/2011_09_26/image_02/data/0000000010.png b/images/KITTI/2011_09_26/image_02/data/0000000010.png new file mode 100644 index 00000000..37707407 Binary files /dev/null and b/images/KITTI/2011_09_26/image_02/data/0000000010.png differ diff --git a/images/KITTI/2011_09_26/image_02/data/0000000011.png b/images/KITTI/2011_09_26/image_02/data/0000000011.png new file mode 100644 index 00000000..2ca5bc97 Binary files /dev/null and b/images/KITTI/2011_09_26/image_02/data/0000000011.png differ diff --git a/images/KITTI/2011_09_26/image_02/data/0000000012.png b/images/KITTI/2011_09_26/image_02/data/0000000012.png new file mode 100644 index 00000000..8379d1c4 Binary files /dev/null and b/images/KITTI/2011_09_26/image_02/data/0000000012.png differ diff --git a/images/KITTI/2011_09_26/image_02/data/0000000013.png b/images/KITTI/2011_09_26/image_02/data/0000000013.png new file mode 100644 index 00000000..c7d6f01b Binary files /dev/null and b/images/KITTI/2011_09_26/image_02/data/0000000013.png differ diff --git a/images/KITTI/2011_09_26/image_02/data/0000000014.png b/images/KITTI/2011_09_26/image_02/data/0000000014.png new file mode 100644 index 00000000..540e9697 Binary files /dev/null and b/images/KITTI/2011_09_26/image_02/data/0000000014.png differ diff --git a/images/KITTI/2011_09_26/image_02/data/0000000015.png b/images/KITTI/2011_09_26/image_02/data/0000000015.png new file mode 100644 index 00000000..ee71fe3a Binary files /dev/null and b/images/KITTI/2011_09_26/image_02/data/0000000015.png differ diff --git a/images/KITTI/2011_09_26/image_02/data/0000000016.png b/images/KITTI/2011_09_26/image_02/data/0000000016.png new file mode 100644 index 00000000..31797b6e Binary files /dev/null and b/images/KITTI/2011_09_26/image_02/data/0000000016.png differ diff --git a/images/KITTI/2011_09_26/image_02/data/0000000017.png b/images/KITTI/2011_09_26/image_02/data/0000000017.png new file mode 100644 index 00000000..b5f5c601 Binary files /dev/null and b/images/KITTI/2011_09_26/image_02/data/0000000017.png differ diff --git a/images/KITTI/2011_09_26/image_02/data/0000000018.png b/images/KITTI/2011_09_26/image_02/data/0000000018.png new file mode 100644 index 00000000..f9f1b1ff Binary files /dev/null and b/images/KITTI/2011_09_26/image_02/data/0000000018.png differ diff --git a/images/KITTI/2011_09_26/image_02/data/0000000019.png b/images/KITTI/2011_09_26/image_02/data/0000000019.png new file mode 100644 index 00000000..cf014351 Binary files /dev/null and b/images/KITTI/2011_09_26/image_02/data/0000000019.png differ diff --git a/images/KITTI/2011_09_26/image_02/data/0000000020.png b/images/KITTI/2011_09_26/image_02/data/0000000020.png new file mode 100644 index 00000000..72444824 Binary files /dev/null and b/images/KITTI/2011_09_26/image_02/data/0000000020.png differ diff --git a/images/KITTI/2011_09_26/image_02/data/0000000021.png b/images/KITTI/2011_09_26/image_02/data/0000000021.png new file mode 100644 index 00000000..99b3171a Binary files /dev/null and b/images/KITTI/2011_09_26/image_02/data/0000000021.png differ diff --git a/images/KITTI/2011_09_26/image_02/data/0000000022.png b/images/KITTI/2011_09_26/image_02/data/0000000022.png new file mode 100644 index 00000000..2e4820e7 Binary files /dev/null and b/images/KITTI/2011_09_26/image_02/data/0000000022.png differ diff --git a/images/KITTI/2011_09_26/image_02/data/0000000023.png b/images/KITTI/2011_09_26/image_02/data/0000000023.png new file mode 100644 index 00000000..caf1f3f1 Binary files /dev/null and b/images/KITTI/2011_09_26/image_02/data/0000000023.png differ diff --git a/images/KITTI/2011_09_26/image_02/data/0000000024.png b/images/KITTI/2011_09_26/image_02/data/0000000024.png new file mode 100644 index 00000000..106c7171 Binary files /dev/null and b/images/KITTI/2011_09_26/image_02/data/0000000024.png differ diff --git a/images/KITTI/2011_09_26/image_02/data/0000000025.png b/images/KITTI/2011_09_26/image_02/data/0000000025.png new file mode 100644 index 00000000..9e93df62 Binary files /dev/null and b/images/KITTI/2011_09_26/image_02/data/0000000025.png differ diff --git a/images/KITTI/2011_09_26/image_02/data/0000000026.png b/images/KITTI/2011_09_26/image_02/data/0000000026.png new file mode 100644 index 00000000..8a2b495d Binary files /dev/null and b/images/KITTI/2011_09_26/image_02/data/0000000026.png differ diff --git a/images/KITTI/2011_09_26/image_02/data/0000000027.png b/images/KITTI/2011_09_26/image_02/data/0000000027.png new file mode 100644 index 00000000..753fe500 Binary files /dev/null and b/images/KITTI/2011_09_26/image_02/data/0000000027.png differ diff --git a/images/KITTI/2011_09_26/image_02/data/0000000028.png b/images/KITTI/2011_09_26/image_02/data/0000000028.png new file mode 100644 index 00000000..3e2e9797 Binary files /dev/null and b/images/KITTI/2011_09_26/image_02/data/0000000028.png differ diff --git a/images/KITTI/2011_09_26/image_02/data/0000000029.png b/images/KITTI/2011_09_26/image_02/data/0000000029.png new file mode 100644 index 00000000..e2766ffc Binary files /dev/null and b/images/KITTI/2011_09_26/image_02/data/0000000029.png differ diff --git a/images/KITTI/2011_09_26/image_02/data/0000000030.png b/images/KITTI/2011_09_26/image_02/data/0000000030.png new file mode 100644 index 00000000..1e499f4f Binary files /dev/null and b/images/KITTI/2011_09_26/image_02/data/0000000030.png differ diff --git a/images/KITTI/2011_09_26/image_02/data/0000000031.png b/images/KITTI/2011_09_26/image_02/data/0000000031.png new file mode 100644 index 00000000..bac9e821 Binary files /dev/null and b/images/KITTI/2011_09_26/image_02/data/0000000031.png differ diff --git a/images/KITTI/2011_09_26/image_02/data/0000000032.png b/images/KITTI/2011_09_26/image_02/data/0000000032.png new file mode 100644 index 00000000..5d2b1bfe Binary files /dev/null and b/images/KITTI/2011_09_26/image_02/data/0000000032.png differ diff --git a/images/KITTI/2011_09_26/image_02/data/0000000033.png b/images/KITTI/2011_09_26/image_02/data/0000000033.png new file mode 100644 index 00000000..15e30d8f Binary files /dev/null and b/images/KITTI/2011_09_26/image_02/data/0000000033.png differ diff --git a/images/KITTI/2011_09_26/image_02/data/0000000034.png b/images/KITTI/2011_09_26/image_02/data/0000000034.png new file mode 100644 index 00000000..0656397f Binary files /dev/null and b/images/KITTI/2011_09_26/image_02/data/0000000034.png differ diff --git a/images/KITTI/2011_09_26/image_02/data/0000000035.png b/images/KITTI/2011_09_26/image_02/data/0000000035.png new file mode 100644 index 00000000..f2cf897d Binary files /dev/null and b/images/KITTI/2011_09_26/image_02/data/0000000035.png differ diff --git a/images/KITTI/2011_09_26/image_02/data/0000000036.png b/images/KITTI/2011_09_26/image_02/data/0000000036.png new file mode 100644 index 00000000..fafc9ed0 Binary files /dev/null and b/images/KITTI/2011_09_26/image_02/data/0000000036.png differ diff --git a/images/KITTI/2011_09_26/image_02/data/0000000037.png b/images/KITTI/2011_09_26/image_02/data/0000000037.png new file mode 100644 index 00000000..2562d344 Binary files /dev/null and b/images/KITTI/2011_09_26/image_02/data/0000000037.png differ diff --git a/images/KITTI/2011_09_26/image_02/data/0000000038.png b/images/KITTI/2011_09_26/image_02/data/0000000038.png new file mode 100644 index 00000000..4dad7f7f Binary files /dev/null and b/images/KITTI/2011_09_26/image_02/data/0000000038.png differ diff --git a/images/KITTI/2011_09_26/image_02/data/0000000039.png b/images/KITTI/2011_09_26/image_02/data/0000000039.png new file mode 100644 index 00000000..6aa9443b Binary files /dev/null and b/images/KITTI/2011_09_26/image_02/data/0000000039.png differ diff --git a/images/KITTI/2011_09_26/image_02/data/0000000040.png b/images/KITTI/2011_09_26/image_02/data/0000000040.png new file mode 100644 index 00000000..10e6c8d6 Binary files /dev/null and b/images/KITTI/2011_09_26/image_02/data/0000000040.png differ diff --git a/images/KITTI/2011_09_26/image_02/data/0000000041.png b/images/KITTI/2011_09_26/image_02/data/0000000041.png new file mode 100644 index 00000000..d2b0f6a4 Binary files /dev/null and b/images/KITTI/2011_09_26/image_02/data/0000000041.png differ diff --git a/images/KITTI/2011_09_26/image_02/data/0000000042.png b/images/KITTI/2011_09_26/image_02/data/0000000042.png new file mode 100644 index 00000000..4982abb0 Binary files /dev/null and b/images/KITTI/2011_09_26/image_02/data/0000000042.png differ diff --git a/images/KITTI/2011_09_26/image_02/data/0000000043.png b/images/KITTI/2011_09_26/image_02/data/0000000043.png new file mode 100644 index 00000000..1b925165 Binary files /dev/null and b/images/KITTI/2011_09_26/image_02/data/0000000043.png differ diff --git a/images/KITTI/2011_09_26/image_02/data/0000000044.png b/images/KITTI/2011_09_26/image_02/data/0000000044.png new file mode 100644 index 00000000..f06660cd Binary files /dev/null and b/images/KITTI/2011_09_26/image_02/data/0000000044.png differ diff --git a/images/KITTI/2011_09_26/image_02/data/0000000045.png b/images/KITTI/2011_09_26/image_02/data/0000000045.png new file mode 100644 index 00000000..8e9f6c13 Binary files /dev/null and b/images/KITTI/2011_09_26/image_02/data/0000000045.png differ diff --git a/images/KITTI/2011_09_26/image_02/data/0000000046.png b/images/KITTI/2011_09_26/image_02/data/0000000046.png new file mode 100644 index 00000000..9bc5c2af Binary files /dev/null and b/images/KITTI/2011_09_26/image_02/data/0000000046.png differ diff --git a/images/KITTI/2011_09_26/image_02/data/0000000047.png b/images/KITTI/2011_09_26/image_02/data/0000000047.png new file mode 100644 index 00000000..62fc3e72 Binary files /dev/null and b/images/KITTI/2011_09_26/image_02/data/0000000047.png differ diff --git a/images/KITTI/2011_09_26/image_02/data/0000000048.png b/images/KITTI/2011_09_26/image_02/data/0000000048.png new file mode 100644 index 00000000..b7542ccc Binary files /dev/null and b/images/KITTI/2011_09_26/image_02/data/0000000048.png differ diff --git a/images/KITTI/2011_09_26/image_02/data/0000000049.png b/images/KITTI/2011_09_26/image_02/data/0000000049.png new file mode 100644 index 00000000..8bb31269 Binary files /dev/null and b/images/KITTI/2011_09_26/image_02/data/0000000049.png differ diff --git a/images/KITTI/2011_09_26/image_02/data/0000000050.png b/images/KITTI/2011_09_26/image_02/data/0000000050.png new file mode 100644 index 00000000..57710c13 Binary files /dev/null and b/images/KITTI/2011_09_26/image_02/data/0000000050.png differ diff --git a/images/KITTI/2011_09_26/image_02/data/0000000051.png b/images/KITTI/2011_09_26/image_02/data/0000000051.png new file mode 100644 index 00000000..9b563215 Binary files /dev/null and b/images/KITTI/2011_09_26/image_02/data/0000000051.png differ diff --git a/images/KITTI/2011_09_26/image_02/data/0000000052.png b/images/KITTI/2011_09_26/image_02/data/0000000052.png new file mode 100644 index 00000000..95129071 Binary files /dev/null and b/images/KITTI/2011_09_26/image_02/data/0000000052.png differ diff --git a/images/KITTI/2011_09_26/image_02/data/0000000053.png b/images/KITTI/2011_09_26/image_02/data/0000000053.png new file mode 100644 index 00000000..90159f0c Binary files /dev/null and b/images/KITTI/2011_09_26/image_02/data/0000000053.png differ diff --git a/images/KITTI/2011_09_26/image_02/data/0000000054.png b/images/KITTI/2011_09_26/image_02/data/0000000054.png new file mode 100644 index 00000000..a4dc7095 Binary files /dev/null and b/images/KITTI/2011_09_26/image_02/data/0000000054.png differ diff --git a/images/KITTI/2011_09_26/image_02/data/0000000055.png b/images/KITTI/2011_09_26/image_02/data/0000000055.png new file mode 100644 index 00000000..f2512214 Binary files /dev/null and b/images/KITTI/2011_09_26/image_02/data/0000000055.png differ diff --git a/images/KITTI/2011_09_26/image_02/data/0000000056.png b/images/KITTI/2011_09_26/image_02/data/0000000056.png new file mode 100644 index 00000000..e59306f0 Binary files /dev/null and b/images/KITTI/2011_09_26/image_02/data/0000000056.png differ diff --git a/images/KITTI/2011_09_26/image_02/data/0000000057.png b/images/KITTI/2011_09_26/image_02/data/0000000057.png new file mode 100644 index 00000000..595cd7cb Binary files /dev/null and b/images/KITTI/2011_09_26/image_02/data/0000000057.png differ diff --git a/images/KITTI/2011_09_26/image_02/data/0000000058.png b/images/KITTI/2011_09_26/image_02/data/0000000058.png new file mode 100644 index 00000000..ab97b5c0 Binary files /dev/null and b/images/KITTI/2011_09_26/image_02/data/0000000058.png differ diff --git a/images/KITTI/2011_09_26/image_02/data/0000000059.png b/images/KITTI/2011_09_26/image_02/data/0000000059.png new file mode 100644 index 00000000..7ef13646 Binary files /dev/null and b/images/KITTI/2011_09_26/image_02/data/0000000059.png differ diff --git a/images/KITTI/2011_09_26/image_02/data/0000000060.png b/images/KITTI/2011_09_26/image_02/data/0000000060.png new file mode 100644 index 00000000..6e4c7566 Binary files /dev/null and b/images/KITTI/2011_09_26/image_02/data/0000000060.png differ diff --git a/images/KITTI/2011_09_26/image_02/data/0000000061.png b/images/KITTI/2011_09_26/image_02/data/0000000061.png new file mode 100644 index 00000000..eb84455d Binary files /dev/null and b/images/KITTI/2011_09_26/image_02/data/0000000061.png differ diff --git a/images/KITTI/2011_09_26/image_02/data/0000000062.png b/images/KITTI/2011_09_26/image_02/data/0000000062.png new file mode 100644 index 00000000..2d694328 Binary files /dev/null and b/images/KITTI/2011_09_26/image_02/data/0000000062.png differ diff --git a/images/KITTI/2011_09_26/image_02/data/0000000063.png b/images/KITTI/2011_09_26/image_02/data/0000000063.png new file mode 100644 index 00000000..0c67f430 Binary files /dev/null and b/images/KITTI/2011_09_26/image_02/data/0000000063.png differ diff --git a/images/KITTI/2011_09_26/image_02/data/0000000064.png b/images/KITTI/2011_09_26/image_02/data/0000000064.png new file mode 100644 index 00000000..e4604867 Binary files /dev/null and b/images/KITTI/2011_09_26/image_02/data/0000000064.png differ diff --git a/images/KITTI/2011_09_26/image_02/data/0000000065.png b/images/KITTI/2011_09_26/image_02/data/0000000065.png new file mode 100644 index 00000000..72688043 Binary files /dev/null and b/images/KITTI/2011_09_26/image_02/data/0000000065.png differ diff --git a/images/KITTI/2011_09_26/image_02/data/0000000066.png b/images/KITTI/2011_09_26/image_02/data/0000000066.png new file mode 100644 index 00000000..2c068ca8 Binary files /dev/null and b/images/KITTI/2011_09_26/image_02/data/0000000066.png differ diff --git a/images/KITTI/2011_09_26/image_02/data/0000000067.png b/images/KITTI/2011_09_26/image_02/data/0000000067.png new file mode 100644 index 00000000..c28409ac Binary files /dev/null and b/images/KITTI/2011_09_26/image_02/data/0000000067.png differ diff --git a/images/KITTI/2011_09_26/image_02/data/0000000068.png b/images/KITTI/2011_09_26/image_02/data/0000000068.png new file mode 100644 index 00000000..b18de490 Binary files /dev/null and b/images/KITTI/2011_09_26/image_02/data/0000000068.png differ diff --git a/images/KITTI/2011_09_26/image_02/data/0000000069.png b/images/KITTI/2011_09_26/image_02/data/0000000069.png new file mode 100644 index 00000000..0cfe083f Binary files /dev/null and b/images/KITTI/2011_09_26/image_02/data/0000000069.png differ diff --git a/images/KITTI/2011_09_26/image_02/data/0000000070.png b/images/KITTI/2011_09_26/image_02/data/0000000070.png new file mode 100644 index 00000000..af17df35 Binary files /dev/null and b/images/KITTI/2011_09_26/image_02/data/0000000070.png differ diff --git a/images/KITTI/2011_09_26/image_02/data/0000000071.png b/images/KITTI/2011_09_26/image_02/data/0000000071.png new file mode 100644 index 00000000..1b3e603b Binary files /dev/null and b/images/KITTI/2011_09_26/image_02/data/0000000071.png differ diff --git a/images/KITTI/2011_09_26/image_02/data/0000000072.png b/images/KITTI/2011_09_26/image_02/data/0000000072.png new file mode 100644 index 00000000..32a2de3f Binary files /dev/null and b/images/KITTI/2011_09_26/image_02/data/0000000072.png differ diff --git a/images/KITTI/2011_09_26/image_02/data/0000000073.png b/images/KITTI/2011_09_26/image_02/data/0000000073.png new file mode 100644 index 00000000..6e9f9540 Binary files /dev/null and b/images/KITTI/2011_09_26/image_02/data/0000000073.png differ diff --git a/images/KITTI/2011_09_26/image_02/data/0000000074.png b/images/KITTI/2011_09_26/image_02/data/0000000074.png new file mode 100644 index 00000000..9334e08d Binary files /dev/null and b/images/KITTI/2011_09_26/image_02/data/0000000074.png differ diff --git a/images/KITTI/2011_09_26/image_02/data/0000000075.png b/images/KITTI/2011_09_26/image_02/data/0000000075.png new file mode 100644 index 00000000..513ebdf5 Binary files /dev/null and b/images/KITTI/2011_09_26/image_02/data/0000000075.png differ diff --git a/images/KITTI/2011_09_26/image_02/data/0000000076.png b/images/KITTI/2011_09_26/image_02/data/0000000076.png new file mode 100644 index 00000000..33d1017d Binary files /dev/null and b/images/KITTI/2011_09_26/image_02/data/0000000076.png differ diff --git a/images/KITTI/2011_09_26/image_02/data/0000000077.png b/images/KITTI/2011_09_26/image_02/data/0000000077.png new file mode 100644 index 00000000..4e738ef4 Binary files /dev/null and b/images/KITTI/2011_09_26/image_02/data/0000000077.png differ diff --git a/images/KITTI/2011_09_26/velodyne_points/data/0000000000.bin b/images/KITTI/2011_09_26/velodyne_points/data/0000000000.bin new file mode 100644 index 00000000..84b1e80e Binary files /dev/null and b/images/KITTI/2011_09_26/velodyne_points/data/0000000000.bin differ diff --git a/images/KITTI/2011_09_26/velodyne_points/data/0000000001.bin b/images/KITTI/2011_09_26/velodyne_points/data/0000000001.bin new file mode 100644 index 00000000..b50f9eb0 Binary files /dev/null and b/images/KITTI/2011_09_26/velodyne_points/data/0000000001.bin differ diff --git a/images/KITTI/2011_09_26/velodyne_points/data/0000000002.bin b/images/KITTI/2011_09_26/velodyne_points/data/0000000002.bin new file mode 100644 index 00000000..1c342095 Binary files /dev/null and b/images/KITTI/2011_09_26/velodyne_points/data/0000000002.bin differ diff --git a/images/KITTI/2011_09_26/velodyne_points/data/0000000003.bin b/images/KITTI/2011_09_26/velodyne_points/data/0000000003.bin new file mode 100644 index 00000000..c42c08f4 Binary files /dev/null and b/images/KITTI/2011_09_26/velodyne_points/data/0000000003.bin differ diff --git a/images/KITTI/2011_09_26/velodyne_points/data/0000000004.bin b/images/KITTI/2011_09_26/velodyne_points/data/0000000004.bin new file mode 100644 index 00000000..cf762f69 Binary files /dev/null and b/images/KITTI/2011_09_26/velodyne_points/data/0000000004.bin differ diff --git a/images/KITTI/2011_09_26/velodyne_points/data/0000000005.bin b/images/KITTI/2011_09_26/velodyne_points/data/0000000005.bin new file mode 100644 index 00000000..8a56cfc1 Binary files /dev/null and b/images/KITTI/2011_09_26/velodyne_points/data/0000000005.bin differ diff --git a/images/KITTI/2011_09_26/velodyne_points/data/0000000006.bin b/images/KITTI/2011_09_26/velodyne_points/data/0000000006.bin new file mode 100644 index 00000000..74c0cfd5 Binary files /dev/null and b/images/KITTI/2011_09_26/velodyne_points/data/0000000006.bin differ diff --git a/images/KITTI/2011_09_26/velodyne_points/data/0000000007.bin b/images/KITTI/2011_09_26/velodyne_points/data/0000000007.bin new file mode 100644 index 00000000..faf41934 Binary files /dev/null and b/images/KITTI/2011_09_26/velodyne_points/data/0000000007.bin differ diff --git a/images/KITTI/2011_09_26/velodyne_points/data/0000000008.bin b/images/KITTI/2011_09_26/velodyne_points/data/0000000008.bin new file mode 100644 index 00000000..29cab109 Binary files /dev/null and b/images/KITTI/2011_09_26/velodyne_points/data/0000000008.bin differ diff --git a/images/KITTI/2011_09_26/velodyne_points/data/0000000009.bin b/images/KITTI/2011_09_26/velodyne_points/data/0000000009.bin new file mode 100644 index 00000000..db7ad459 Binary files /dev/null and b/images/KITTI/2011_09_26/velodyne_points/data/0000000009.bin differ diff --git a/images/KITTI/2011_09_26/velodyne_points/data/0000000010.bin b/images/KITTI/2011_09_26/velodyne_points/data/0000000010.bin new file mode 100644 index 00000000..dc05e41c Binary files /dev/null and b/images/KITTI/2011_09_26/velodyne_points/data/0000000010.bin differ diff --git a/images/KITTI/2011_09_26/velodyne_points/data/0000000011.bin b/images/KITTI/2011_09_26/velodyne_points/data/0000000011.bin new file mode 100644 index 00000000..0883fe4d Binary files /dev/null and b/images/KITTI/2011_09_26/velodyne_points/data/0000000011.bin differ diff --git a/images/KITTI/2011_09_26/velodyne_points/data/0000000012.bin b/images/KITTI/2011_09_26/velodyne_points/data/0000000012.bin new file mode 100644 index 00000000..20576789 Binary files /dev/null and b/images/KITTI/2011_09_26/velodyne_points/data/0000000012.bin differ diff --git a/images/KITTI/2011_09_26/velodyne_points/data/0000000013.bin b/images/KITTI/2011_09_26/velodyne_points/data/0000000013.bin new file mode 100644 index 00000000..a1b298fe Binary files /dev/null and b/images/KITTI/2011_09_26/velodyne_points/data/0000000013.bin differ diff --git a/images/KITTI/2011_09_26/velodyne_points/data/0000000014.bin b/images/KITTI/2011_09_26/velodyne_points/data/0000000014.bin new file mode 100644 index 00000000..75eb168f Binary files /dev/null and b/images/KITTI/2011_09_26/velodyne_points/data/0000000014.bin differ diff --git a/images/KITTI/2011_09_26/velodyne_points/data/0000000015.bin b/images/KITTI/2011_09_26/velodyne_points/data/0000000015.bin new file mode 100644 index 00000000..29e89791 Binary files /dev/null and b/images/KITTI/2011_09_26/velodyne_points/data/0000000015.bin differ diff --git a/images/KITTI/2011_09_26/velodyne_points/data/0000000016.bin b/images/KITTI/2011_09_26/velodyne_points/data/0000000016.bin new file mode 100644 index 00000000..a25bae51 Binary files /dev/null and b/images/KITTI/2011_09_26/velodyne_points/data/0000000016.bin differ diff --git a/images/KITTI/2011_09_26/velodyne_points/data/0000000017.bin b/images/KITTI/2011_09_26/velodyne_points/data/0000000017.bin new file mode 100644 index 00000000..7d5a7565 Binary files /dev/null and b/images/KITTI/2011_09_26/velodyne_points/data/0000000017.bin differ diff --git a/images/KITTI/2011_09_26/velodyne_points/data/0000000018.bin b/images/KITTI/2011_09_26/velodyne_points/data/0000000018.bin new file mode 100644 index 00000000..7eb5a8bf Binary files /dev/null and b/images/KITTI/2011_09_26/velodyne_points/data/0000000018.bin differ diff --git a/images/KITTI/2011_09_26/velodyne_points/data/0000000019.bin b/images/KITTI/2011_09_26/velodyne_points/data/0000000019.bin new file mode 100644 index 00000000..fa9c6783 Binary files /dev/null and b/images/KITTI/2011_09_26/velodyne_points/data/0000000019.bin differ diff --git a/images/KITTI/2011_09_26/velodyne_points/data/0000000020.bin b/images/KITTI/2011_09_26/velodyne_points/data/0000000020.bin new file mode 100644 index 00000000..d73a0cda Binary files /dev/null and b/images/KITTI/2011_09_26/velodyne_points/data/0000000020.bin differ diff --git a/images/KITTI/2011_09_26/velodyne_points/data/0000000021.bin b/images/KITTI/2011_09_26/velodyne_points/data/0000000021.bin new file mode 100644 index 00000000..e0245f09 Binary files /dev/null and b/images/KITTI/2011_09_26/velodyne_points/data/0000000021.bin differ diff --git a/images/KITTI/2011_09_26/velodyne_points/data/0000000022.bin b/images/KITTI/2011_09_26/velodyne_points/data/0000000022.bin new file mode 100644 index 00000000..1b6f9a2b Binary files /dev/null and b/images/KITTI/2011_09_26/velodyne_points/data/0000000022.bin differ diff --git a/images/KITTI/2011_09_26/velodyne_points/data/0000000023.bin b/images/KITTI/2011_09_26/velodyne_points/data/0000000023.bin new file mode 100644 index 00000000..435572d5 Binary files /dev/null and b/images/KITTI/2011_09_26/velodyne_points/data/0000000023.bin differ diff --git a/images/KITTI/2011_09_26/velodyne_points/data/0000000024.bin b/images/KITTI/2011_09_26/velodyne_points/data/0000000024.bin new file mode 100644 index 00000000..512fabdd Binary files /dev/null and b/images/KITTI/2011_09_26/velodyne_points/data/0000000024.bin differ diff --git a/images/KITTI/2011_09_26/velodyne_points/data/0000000025.bin b/images/KITTI/2011_09_26/velodyne_points/data/0000000025.bin new file mode 100644 index 00000000..aaf15851 Binary files /dev/null and b/images/KITTI/2011_09_26/velodyne_points/data/0000000025.bin differ diff --git a/images/KITTI/2011_09_26/velodyne_points/data/0000000026.bin b/images/KITTI/2011_09_26/velodyne_points/data/0000000026.bin new file mode 100644 index 00000000..8f0d3350 Binary files /dev/null and b/images/KITTI/2011_09_26/velodyne_points/data/0000000026.bin differ diff --git a/images/KITTI/2011_09_26/velodyne_points/data/0000000027.bin b/images/KITTI/2011_09_26/velodyne_points/data/0000000027.bin new file mode 100644 index 00000000..80ed2310 Binary files /dev/null and b/images/KITTI/2011_09_26/velodyne_points/data/0000000027.bin differ diff --git a/images/KITTI/2011_09_26/velodyne_points/data/0000000028.bin b/images/KITTI/2011_09_26/velodyne_points/data/0000000028.bin new file mode 100644 index 00000000..055c6746 Binary files /dev/null and b/images/KITTI/2011_09_26/velodyne_points/data/0000000028.bin differ diff --git a/images/KITTI/2011_09_26/velodyne_points/data/0000000029.bin b/images/KITTI/2011_09_26/velodyne_points/data/0000000029.bin new file mode 100644 index 00000000..6b3974ee Binary files /dev/null and b/images/KITTI/2011_09_26/velodyne_points/data/0000000029.bin differ diff --git a/images/KITTI/2011_09_26/velodyne_points/data/0000000030.bin b/images/KITTI/2011_09_26/velodyne_points/data/0000000030.bin new file mode 100644 index 00000000..7ade08ea Binary files /dev/null and b/images/KITTI/2011_09_26/velodyne_points/data/0000000030.bin differ diff --git a/images/KITTI/2011_09_26/velodyne_points/data/0000000031.bin b/images/KITTI/2011_09_26/velodyne_points/data/0000000031.bin new file mode 100644 index 00000000..67d8cf84 Binary files /dev/null and b/images/KITTI/2011_09_26/velodyne_points/data/0000000031.bin differ diff --git a/images/KITTI/2011_09_26/velodyne_points/data/0000000032.bin b/images/KITTI/2011_09_26/velodyne_points/data/0000000032.bin new file mode 100644 index 00000000..0e133e54 Binary files /dev/null and b/images/KITTI/2011_09_26/velodyne_points/data/0000000032.bin differ diff --git a/images/KITTI/2011_09_26/velodyne_points/data/0000000033.bin b/images/KITTI/2011_09_26/velodyne_points/data/0000000033.bin new file mode 100644 index 00000000..761a4d9e Binary files /dev/null and b/images/KITTI/2011_09_26/velodyne_points/data/0000000033.bin differ diff --git a/images/KITTI/2011_09_26/velodyne_points/data/0000000034.bin b/images/KITTI/2011_09_26/velodyne_points/data/0000000034.bin new file mode 100644 index 00000000..9043901f Binary files /dev/null and b/images/KITTI/2011_09_26/velodyne_points/data/0000000034.bin differ diff --git a/images/KITTI/2011_09_26/velodyne_points/data/0000000035.bin b/images/KITTI/2011_09_26/velodyne_points/data/0000000035.bin new file mode 100644 index 00000000..3f83bf2b Binary files /dev/null and b/images/KITTI/2011_09_26/velodyne_points/data/0000000035.bin differ diff --git a/images/KITTI/2011_09_26/velodyne_points/data/0000000036.bin b/images/KITTI/2011_09_26/velodyne_points/data/0000000036.bin new file mode 100644 index 00000000..e97b0421 Binary files /dev/null and b/images/KITTI/2011_09_26/velodyne_points/data/0000000036.bin differ diff --git a/images/KITTI/2011_09_26/velodyne_points/data/0000000037.bin b/images/KITTI/2011_09_26/velodyne_points/data/0000000037.bin new file mode 100644 index 00000000..5f52317d Binary files /dev/null and b/images/KITTI/2011_09_26/velodyne_points/data/0000000037.bin differ diff --git a/images/KITTI/2011_09_26/velodyne_points/data/0000000038.bin b/images/KITTI/2011_09_26/velodyne_points/data/0000000038.bin new file mode 100644 index 00000000..f22e4a05 Binary files /dev/null and b/images/KITTI/2011_09_26/velodyne_points/data/0000000038.bin differ diff --git a/images/KITTI/2011_09_26/velodyne_points/data/0000000039.bin b/images/KITTI/2011_09_26/velodyne_points/data/0000000039.bin new file mode 100644 index 00000000..997bc4ab Binary files /dev/null and b/images/KITTI/2011_09_26/velodyne_points/data/0000000039.bin differ diff --git a/images/KITTI/2011_09_26/velodyne_points/data/0000000040.bin b/images/KITTI/2011_09_26/velodyne_points/data/0000000040.bin new file mode 100644 index 00000000..f5fca443 Binary files /dev/null and b/images/KITTI/2011_09_26/velodyne_points/data/0000000040.bin differ diff --git a/images/KITTI/2011_09_26/velodyne_points/data/0000000041.bin b/images/KITTI/2011_09_26/velodyne_points/data/0000000041.bin new file mode 100644 index 00000000..4d1e270a Binary files /dev/null and b/images/KITTI/2011_09_26/velodyne_points/data/0000000041.bin differ diff --git a/images/KITTI/2011_09_26/velodyne_points/data/0000000042.bin b/images/KITTI/2011_09_26/velodyne_points/data/0000000042.bin new file mode 100644 index 00000000..e14e2282 Binary files /dev/null and b/images/KITTI/2011_09_26/velodyne_points/data/0000000042.bin differ diff --git a/images/KITTI/2011_09_26/velodyne_points/data/0000000043.bin b/images/KITTI/2011_09_26/velodyne_points/data/0000000043.bin new file mode 100644 index 00000000..2018d930 Binary files /dev/null and b/images/KITTI/2011_09_26/velodyne_points/data/0000000043.bin differ diff --git a/images/KITTI/2011_09_26/velodyne_points/data/0000000044.bin b/images/KITTI/2011_09_26/velodyne_points/data/0000000044.bin new file mode 100644 index 00000000..f9f37bd8 Binary files /dev/null and b/images/KITTI/2011_09_26/velodyne_points/data/0000000044.bin differ diff --git a/images/KITTI/2011_09_26/velodyne_points/data/0000000045.bin b/images/KITTI/2011_09_26/velodyne_points/data/0000000045.bin new file mode 100644 index 00000000..c94b686e Binary files /dev/null and b/images/KITTI/2011_09_26/velodyne_points/data/0000000045.bin differ diff --git a/images/KITTI/2011_09_26/velodyne_points/data/0000000046.bin b/images/KITTI/2011_09_26/velodyne_points/data/0000000046.bin new file mode 100644 index 00000000..b270eba2 Binary files /dev/null and b/images/KITTI/2011_09_26/velodyne_points/data/0000000046.bin differ diff --git a/images/KITTI/2011_09_26/velodyne_points/data/0000000047.bin b/images/KITTI/2011_09_26/velodyne_points/data/0000000047.bin new file mode 100644 index 00000000..ca3ffa64 Binary files /dev/null and b/images/KITTI/2011_09_26/velodyne_points/data/0000000047.bin differ diff --git a/images/KITTI/2011_09_26/velodyne_points/data/0000000048.bin b/images/KITTI/2011_09_26/velodyne_points/data/0000000048.bin new file mode 100644 index 00000000..cfc16a9a Binary files /dev/null and b/images/KITTI/2011_09_26/velodyne_points/data/0000000048.bin differ diff --git a/images/KITTI/2011_09_26/velodyne_points/data/0000000049.bin b/images/KITTI/2011_09_26/velodyne_points/data/0000000049.bin new file mode 100644 index 00000000..b33d55a4 Binary files /dev/null and b/images/KITTI/2011_09_26/velodyne_points/data/0000000049.bin differ diff --git a/images/KITTI/2011_09_26/velodyne_points/data/0000000050.bin b/images/KITTI/2011_09_26/velodyne_points/data/0000000050.bin new file mode 100644 index 00000000..667f9d78 Binary files /dev/null and b/images/KITTI/2011_09_26/velodyne_points/data/0000000050.bin differ diff --git a/images/KITTI/2011_09_26/velodyne_points/data/0000000051.bin b/images/KITTI/2011_09_26/velodyne_points/data/0000000051.bin new file mode 100644 index 00000000..aa62c03e Binary files /dev/null and b/images/KITTI/2011_09_26/velodyne_points/data/0000000051.bin differ diff --git a/images/KITTI/2011_09_26/velodyne_points/data/0000000052.bin b/images/KITTI/2011_09_26/velodyne_points/data/0000000052.bin new file mode 100644 index 00000000..da76984d Binary files /dev/null and b/images/KITTI/2011_09_26/velodyne_points/data/0000000052.bin differ diff --git a/images/KITTI/2011_09_26/velodyne_points/data/0000000053.bin b/images/KITTI/2011_09_26/velodyne_points/data/0000000053.bin new file mode 100644 index 00000000..b25b3595 Binary files /dev/null and b/images/KITTI/2011_09_26/velodyne_points/data/0000000053.bin differ diff --git a/images/KITTI/2011_09_26/velodyne_points/data/0000000054.bin b/images/KITTI/2011_09_26/velodyne_points/data/0000000054.bin new file mode 100644 index 00000000..69534d55 Binary files /dev/null and b/images/KITTI/2011_09_26/velodyne_points/data/0000000054.bin differ diff --git a/images/KITTI/2011_09_26/velodyne_points/data/0000000055.bin b/images/KITTI/2011_09_26/velodyne_points/data/0000000055.bin new file mode 100644 index 00000000..75f63649 Binary files /dev/null and b/images/KITTI/2011_09_26/velodyne_points/data/0000000055.bin differ diff --git a/images/KITTI/2011_09_26/velodyne_points/data/0000000056.bin b/images/KITTI/2011_09_26/velodyne_points/data/0000000056.bin new file mode 100644 index 00000000..e5e5097b Binary files /dev/null and b/images/KITTI/2011_09_26/velodyne_points/data/0000000056.bin differ diff --git a/images/KITTI/2011_09_26/velodyne_points/data/0000000057.bin b/images/KITTI/2011_09_26/velodyne_points/data/0000000057.bin new file mode 100644 index 00000000..742a6e8a Binary files /dev/null and b/images/KITTI/2011_09_26/velodyne_points/data/0000000057.bin differ diff --git a/images/KITTI/2011_09_26/velodyne_points/data/0000000058.bin b/images/KITTI/2011_09_26/velodyne_points/data/0000000058.bin new file mode 100644 index 00000000..f60c48ac Binary files /dev/null and b/images/KITTI/2011_09_26/velodyne_points/data/0000000058.bin differ diff --git a/images/KITTI/2011_09_26/velodyne_points/data/0000000059.bin b/images/KITTI/2011_09_26/velodyne_points/data/0000000059.bin new file mode 100644 index 00000000..91591667 Binary files /dev/null and b/images/KITTI/2011_09_26/velodyne_points/data/0000000059.bin differ diff --git a/images/KITTI/2011_09_26/velodyne_points/data/0000000060.bin b/images/KITTI/2011_09_26/velodyne_points/data/0000000060.bin new file mode 100644 index 00000000..f5581e8d Binary files /dev/null and b/images/KITTI/2011_09_26/velodyne_points/data/0000000060.bin differ diff --git a/images/KITTI/2011_09_26/velodyne_points/data/0000000061.bin b/images/KITTI/2011_09_26/velodyne_points/data/0000000061.bin new file mode 100644 index 00000000..9958b06e Binary files /dev/null and b/images/KITTI/2011_09_26/velodyne_points/data/0000000061.bin differ diff --git a/images/KITTI/2011_09_26/velodyne_points/data/0000000062.bin b/images/KITTI/2011_09_26/velodyne_points/data/0000000062.bin new file mode 100644 index 00000000..7f0665b1 Binary files /dev/null and b/images/KITTI/2011_09_26/velodyne_points/data/0000000062.bin differ diff --git a/images/KITTI/2011_09_26/velodyne_points/data/0000000063.bin b/images/KITTI/2011_09_26/velodyne_points/data/0000000063.bin new file mode 100644 index 00000000..97c71f83 Binary files /dev/null and b/images/KITTI/2011_09_26/velodyne_points/data/0000000063.bin differ diff --git a/images/KITTI/2011_09_26/velodyne_points/data/0000000064.bin b/images/KITTI/2011_09_26/velodyne_points/data/0000000064.bin new file mode 100644 index 00000000..05b6c1bb Binary files /dev/null and b/images/KITTI/2011_09_26/velodyne_points/data/0000000064.bin differ diff --git a/images/KITTI/2011_09_26/velodyne_points/data/0000000065.bin b/images/KITTI/2011_09_26/velodyne_points/data/0000000065.bin new file mode 100644 index 00000000..bb4c8ff3 Binary files /dev/null and b/images/KITTI/2011_09_26/velodyne_points/data/0000000065.bin differ diff --git a/images/KITTI/2011_09_26/velodyne_points/data/0000000066.bin b/images/KITTI/2011_09_26/velodyne_points/data/0000000066.bin new file mode 100644 index 00000000..2b6dfc62 Binary files /dev/null and b/images/KITTI/2011_09_26/velodyne_points/data/0000000066.bin differ diff --git a/images/KITTI/2011_09_26/velodyne_points/data/0000000067.bin b/images/KITTI/2011_09_26/velodyne_points/data/0000000067.bin new file mode 100644 index 00000000..5d98d7a0 Binary files /dev/null and b/images/KITTI/2011_09_26/velodyne_points/data/0000000067.bin differ diff --git a/images/KITTI/2011_09_26/velodyne_points/data/0000000068.bin b/images/KITTI/2011_09_26/velodyne_points/data/0000000068.bin new file mode 100644 index 00000000..15cb119e Binary files /dev/null and b/images/KITTI/2011_09_26/velodyne_points/data/0000000068.bin differ diff --git a/images/KITTI/2011_09_26/velodyne_points/data/0000000069.bin b/images/KITTI/2011_09_26/velodyne_points/data/0000000069.bin new file mode 100644 index 00000000..d0a1c1d6 Binary files /dev/null and b/images/KITTI/2011_09_26/velodyne_points/data/0000000069.bin differ diff --git a/images/KITTI/2011_09_26/velodyne_points/data/0000000070.bin b/images/KITTI/2011_09_26/velodyne_points/data/0000000070.bin new file mode 100644 index 00000000..f78f1f58 Binary files /dev/null and b/images/KITTI/2011_09_26/velodyne_points/data/0000000070.bin differ diff --git a/images/KITTI/2011_09_26/velodyne_points/data/0000000071.bin b/images/KITTI/2011_09_26/velodyne_points/data/0000000071.bin new file mode 100644 index 00000000..4d364f64 Binary files /dev/null and b/images/KITTI/2011_09_26/velodyne_points/data/0000000071.bin differ diff --git a/images/KITTI/2011_09_26/velodyne_points/data/0000000072.bin b/images/KITTI/2011_09_26/velodyne_points/data/0000000072.bin new file mode 100644 index 00000000..10f3bac6 Binary files /dev/null and b/images/KITTI/2011_09_26/velodyne_points/data/0000000072.bin differ diff --git a/images/KITTI/2011_09_26/velodyne_points/data/0000000073.bin b/images/KITTI/2011_09_26/velodyne_points/data/0000000073.bin new file mode 100644 index 00000000..150f8d99 Binary files /dev/null and b/images/KITTI/2011_09_26/velodyne_points/data/0000000073.bin differ diff --git a/images/KITTI/2011_09_26/velodyne_points/data/0000000074.bin b/images/KITTI/2011_09_26/velodyne_points/data/0000000074.bin new file mode 100644 index 00000000..0f75bba8 Binary files /dev/null and b/images/KITTI/2011_09_26/velodyne_points/data/0000000074.bin differ diff --git a/images/KITTI/2011_09_26/velodyne_points/data/0000000075.bin b/images/KITTI/2011_09_26/velodyne_points/data/0000000075.bin new file mode 100644 index 00000000..2a75c7d1 Binary files /dev/null and b/images/KITTI/2011_09_26/velodyne_points/data/0000000075.bin differ diff --git a/images/KITTI/2011_09_26/velodyne_points/data/0000000076.bin b/images/KITTI/2011_09_26/velodyne_points/data/0000000076.bin new file mode 100644 index 00000000..3665e4a6 Binary files /dev/null and b/images/KITTI/2011_09_26/velodyne_points/data/0000000076.bin differ diff --git a/images/KITTI/2011_09_26/velodyne_points/data/0000000077.bin b/images/KITTI/2011_09_26/velodyne_points/data/0000000077.bin new file mode 100644 index 00000000..45327898 Binary files /dev/null and b/images/KITTI/2011_09_26/velodyne_points/data/0000000077.bin differ diff --git a/src/FinalProject_Camera.cpp b/src/FinalProject_Camera.cpp new file mode 100644 index 00000000..fd222e89 --- /dev/null +++ b/src/FinalProject_Camera.cpp @@ -0,0 +1,297 @@ + +/* INCLUDES FOR THIS PROJECT */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "dataStructures.h" +#include "matching2D.hpp" +#include "objectDetection2D.hpp" +#include "lidarData.hpp" +#include "camFusion.hpp" + +using namespace std; + +/* MAIN PROGRAM */ +int main(int argc, const char *argv[]) +{ + /* INIT VARIABLES AND DATA STRUCTURES */ + + // data location + string dataPath = "../"; + + // camera + string imgBasePath = dataPath + "images/"; + string imgPrefix = "KITTI/2011_09_26/image_02/data/000000"; // left camera, color + string imgFileType = ".png"; + int imgStartIndex = 0; // first file index to load (assumes Lidar and camera names have identical naming convention) + int imgEndIndex = 18; // last file index to load + int imgStepWidth = 1; + int imgFillWidth = 4; // no. of digits which make up the file index (e.g. img-0001.png) + + // object detection + string yoloBasePath = dataPath + "dat/yolo/"; + string yoloClassesFile = yoloBasePath + "coco.names"; + string yoloModelConfiguration = yoloBasePath + "yolov3.cfg"; + string yoloModelWeights = yoloBasePath + "yolov3.weights"; + + // Lidar + string lidarPrefix = "KITTI/2011_09_26/velodyne_points/data/000000"; + string lidarFileType = ".bin"; + + // calibration data for camera and lidar + cv::Mat P_rect_00(3,4,cv::DataType::type); // 3x4 projection matrix after rectification + cv::Mat R_rect_00(4,4,cv::DataType::type); // 3x3 rectifying rotation to make image planes co-planar + cv::Mat RT(4,4,cv::DataType::type); // rotation matrix and translation vector + + RT.at(0,0) = 7.533745e-03; RT.at(0,1) = -9.999714e-01; RT.at(0,2) = -6.166020e-04; RT.at(0,3) = -4.069766e-03; + RT.at(1,0) = 1.480249e-02; RT.at(1,1) = 7.280733e-04; RT.at(1,2) = -9.998902e-01; RT.at(1,3) = -7.631618e-02; + RT.at(2,0) = 9.998621e-01; RT.at(2,1) = 7.523790e-03; RT.at(2,2) = 1.480755e-02; RT.at(2,3) = -2.717806e-01; + RT.at(3,0) = 0.0; RT.at(3,1) = 0.0; RT.at(3,2) = 0.0; RT.at(3,3) = 1.0; + + R_rect_00.at(0,0) = 9.999239e-01; R_rect_00.at(0,1) = 9.837760e-03; R_rect_00.at(0,2) = -7.445048e-03; R_rect_00.at(0,3) = 0.0; + R_rect_00.at(1,0) = -9.869795e-03; R_rect_00.at(1,1) = 9.999421e-01; R_rect_00.at(1,2) = -4.278459e-03; R_rect_00.at(1,3) = 0.0; + R_rect_00.at(2,0) = 7.402527e-03; R_rect_00.at(2,1) = 4.351614e-03; R_rect_00.at(2,2) = 9.999631e-01; R_rect_00.at(2,3) = 0.0; + R_rect_00.at(3,0) = 0; R_rect_00.at(3,1) = 0; R_rect_00.at(3,2) = 0; R_rect_00.at(3,3) = 1; + + P_rect_00.at(0,0) = 7.215377e+02; P_rect_00.at(0,1) = 0.000000e+00; P_rect_00.at(0,2) = 6.095593e+02; P_rect_00.at(0,3) = 0.000000e+00; + P_rect_00.at(1,0) = 0.000000e+00; P_rect_00.at(1,1) = 7.215377e+02; P_rect_00.at(1,2) = 1.728540e+02; P_rect_00.at(1,3) = 0.000000e+00; + P_rect_00.at(2,0) = 0.000000e+00; P_rect_00.at(2,1) = 0.000000e+00; P_rect_00.at(2,2) = 1.000000e+00; P_rect_00.at(2,3) = 0.000000e+00; + + // misc + double sensorFrameRate = 10.0 / imgStepWidth; // frames per second for Lidar and camera + int dataBufferSize = 2; // no. of images which are held in memory (ring buffer) at the same time + vector dataBuffer; // list of data frames which are held in memory at the same time + bool bVis = false; // visualize results + + /* MAIN LOOP OVER ALL IMAGES */ + + for (size_t imgIndex = 0; imgIndex <= imgEndIndex - imgStartIndex; imgIndex+=imgStepWidth) + { + /* LOAD IMAGE INTO BUFFER */ + + // assemble filenames for current index + ostringstream imgNumber; + imgNumber << setfill('0') << setw(imgFillWidth) << imgStartIndex + imgIndex; + string imgFullFilename = imgBasePath + imgPrefix + imgNumber.str() + imgFileType; + + // load image from file + cv::Mat img = cv::imread(imgFullFilename); + + // push image into data frame buffer + DataFrame frame; + frame.cameraImg = img; + dataBuffer.push_back(frame); + + cout << "#1 : LOAD IMAGE INTO BUFFER done" << endl; + + + /* DETECT & CLASSIFY OBJECTS */ + + float confThreshold = 0.2; + float nmsThreshold = 0.4; + detectObjects((dataBuffer.end() - 1)->cameraImg, (dataBuffer.end() - 1)->boundingBoxes, confThreshold, nmsThreshold, + yoloBasePath, yoloClassesFile, yoloModelConfiguration, yoloModelWeights, bVis); + + cout << "#2 : DETECT & CLASSIFY OBJECTS done" << endl; + + + /* CROP LIDAR POINTS */ + + // load 3D Lidar points from file + string lidarFullFilename = imgBasePath + lidarPrefix + imgNumber.str() + lidarFileType; + std::vector lidarPoints; + loadLidarFromFile(lidarPoints, lidarFullFilename); + + // remove Lidar points based on distance properties + float minZ = -1.5, maxZ = -0.9, minX = 2.0, maxX = 20.0, maxY = 2.0, minR = 0.1; // focus on ego lane + cropLidarPoints(lidarPoints, minX, maxX, maxY, minZ, maxZ, minR); + + (dataBuffer.end() - 1)->lidarPoints = lidarPoints; + + cout << "#3 : CROP LIDAR POINTS done" << endl; + + + /* CLUSTER LIDAR POINT CLOUD */ + + // associate Lidar points with camera-based ROI + float shrinkFactor = 0.10; // shrinks each bounding box by the given percentage to avoid 3D object merging at the edges of an ROI + clusterLidarWithROI((dataBuffer.end()-1)->boundingBoxes, (dataBuffer.end() - 1)->lidarPoints, shrinkFactor, P_rect_00, R_rect_00, RT); + + // Visualize 3D objects + bVis = true; + if(bVis) + { + show3DObjects((dataBuffer.end()-1)->boundingBoxes, cv::Size(4.0, 20.0), cv::Size(2000, 2000), true); + } + bVis = false; + + cout << "#4 : CLUSTER LIDAR POINT CLOUD done" << endl; + + + // REMOVE THIS LINE BEFORE PROCEEDING WITH THE FINAL PROJECT + continue; // skips directly to the next image without processing what comes beneath + + /* DETECT IMAGE KEYPOINTS */ + + // convert current image to grayscale + cv::Mat imgGray; + cv::cvtColor((dataBuffer.end()-1)->cameraImg, imgGray, cv::COLOR_BGR2GRAY); + + // extract 2D keypoints from current image + vector keypoints; // create empty feature list for current image + string detectorType = "SHITOMASI"; + + if (detectorType.compare("SHITOMASI") == 0) + { + detKeypointsShiTomasi(keypoints, imgGray, false); + } + else + { + //... + } + + // optional : limit number of keypoints (helpful for debugging and learning) + bool bLimitKpts = false; + if (bLimitKpts) + { + int maxKeypoints = 50; + + if (detectorType.compare("SHITOMASI") == 0) + { // there is no response info, so keep the first 50 as they are sorted in descending quality order + keypoints.erase(keypoints.begin() + maxKeypoints, keypoints.end()); + } + cv::KeyPointsFilter::retainBest(keypoints, maxKeypoints); + cout << " NOTE: Keypoints have been limited!" << endl; + } + + // push keypoints and descriptor for current frame to end of data buffer + (dataBuffer.end() - 1)->keypoints = keypoints; + + cout << "#5 : DETECT KEYPOINTS done" << endl; + + + /* EXTRACT KEYPOINT DESCRIPTORS */ + + cv::Mat descriptors; + string descriptorType = "BRISK"; // BRISK, BRIEF, ORB, FREAK, AKAZE, SIFT + descKeypoints((dataBuffer.end() - 1)->keypoints, (dataBuffer.end() - 1)->cameraImg, descriptors, descriptorType); + + // push descriptors for current frame to end of data buffer + (dataBuffer.end() - 1)->descriptors = descriptors; + + cout << "#6 : EXTRACT DESCRIPTORS done" << endl; + + + if (dataBuffer.size() > 1) // wait until at least two images have been processed + { + + /* MATCH KEYPOINT DESCRIPTORS */ + + vector matches; + string matcherType = "MAT_BF"; // MAT_BF, MAT_FLANN + string descriptorType = "DES_BINARY"; // DES_BINARY, DES_HOG + string selectorType = "SEL_NN"; // SEL_NN, SEL_KNN + + matchDescriptors((dataBuffer.end() - 2)->keypoints, (dataBuffer.end() - 1)->keypoints, + (dataBuffer.end() - 2)->descriptors, (dataBuffer.end() - 1)->descriptors, + matches, descriptorType, matcherType, selectorType); + + // store matches in current data frame + (dataBuffer.end() - 1)->kptMatches = matches; + + cout << "#7 : MATCH KEYPOINT DESCRIPTORS done" << endl; + + + /* TRACK 3D OBJECT BOUNDING BOXES */ + + //// STUDENT ASSIGNMENT + //// TASK FP.1 -> match list of 3D objects (vector) between current and previous frame (implement ->matchBoundingBoxes) + map bbBestMatches; + matchBoundingBoxes(matches, bbBestMatches, *(dataBuffer.end()-2), *(dataBuffer.end()-1)); // associate bounding boxes between current and previous frame using keypoint matches + //// EOF STUDENT ASSIGNMENT + + // store matches in current data frame + (dataBuffer.end()-1)->bbMatches = bbBestMatches; + + cout << "#8 : TRACK 3D OBJECT BOUNDING BOXES done" << endl; + + + /* COMPUTE TTC ON OBJECT IN FRONT */ + + // loop over all BB match pairs + for (auto it1 = (dataBuffer.end() - 1)->bbMatches.begin(); it1 != (dataBuffer.end() - 1)->bbMatches.end(); ++it1) + { + // find bounding boxes associates with current match + BoundingBox *prevBB, *currBB; + for (auto it2 = (dataBuffer.end() - 1)->boundingBoxes.begin(); it2 != (dataBuffer.end() - 1)->boundingBoxes.end(); ++it2) + { + if (it1->second == it2->boxID) // check wether current match partner corresponds to this BB + { + currBB = &(*it2); + } + } + + for (auto it2 = (dataBuffer.end() - 2)->boundingBoxes.begin(); it2 != (dataBuffer.end() - 2)->boundingBoxes.end(); ++it2) + { + if (it1->first == it2->boxID) // check wether current match partner corresponds to this BB + { + prevBB = &(*it2); + } + } + + // compute TTC for current match + if( currBB->lidarPoints.size()>0 && prevBB->lidarPoints.size()>0 ) // only compute TTC if we have Lidar points + { + //// STUDENT ASSIGNMENT + //// TASK FP.2 -> compute time-to-collision based on Lidar data (implement -> computeTTCLidar) + double ttcLidar; + computeTTCLidar(prevBB->lidarPoints, currBB->lidarPoints, sensorFrameRate, ttcLidar); + //// EOF STUDENT ASSIGNMENT + + //// STUDENT ASSIGNMENT + //// TASK FP.3 -> assign enclosed keypoint matches to bounding box (implement -> clusterKptMatchesWithROI) + //// TASK FP.4 -> compute time-to-collision based on camera (implement -> computeTTCCamera) + double ttcCamera; + clusterKptMatchesWithROI(*currBB, (dataBuffer.end() - 2)->keypoints, (dataBuffer.end() - 1)->keypoints, (dataBuffer.end() - 1)->kptMatches); + computeTTCCamera((dataBuffer.end() - 2)->keypoints, (dataBuffer.end() - 1)->keypoints, currBB->kptMatches, sensorFrameRate, ttcCamera); + //// EOF STUDENT ASSIGNMENT + + bVis = true; + if (bVis) + { + cv::Mat visImg = (dataBuffer.end() - 1)->cameraImg.clone(); + showLidarImgOverlay(visImg, currBB->lidarPoints, P_rect_00, R_rect_00, RT, &visImg); + cv::rectangle(visImg, cv::Point(currBB->roi.x, currBB->roi.y), cv::Point(currBB->roi.x + currBB->roi.width, currBB->roi.y + currBB->roi.height), cv::Scalar(0, 255, 0), 2); + + char str[200]; + sprintf(str, "TTC Lidar : %f s, TTC Camera : %f s", ttcLidar, ttcCamera); + putText(visImg, str, cv::Point2f(80, 50), cv::FONT_HERSHEY_PLAIN, 2, cv::Scalar(0,0,255)); + + string windowName = "Final Results : TTC"; + cv::namedWindow(windowName, 4); + cv::imshow(windowName, visImg); + cout << "Press key to continue to next frame" << endl; + cv::waitKey(0); + } + bVis = false; + + } // eof TTC computation + } // eof loop over all BB matches + + } + + } // eof loop over all images + + return 0; +} diff --git a/src/camFusion.hpp b/src/camFusion.hpp new file mode 100644 index 00000000..eca20a51 --- /dev/null +++ b/src/camFusion.hpp @@ -0,0 +1,21 @@ + +#ifndef camFusion_hpp +#define camFusion_hpp + +#include +#include +#include +#include "dataStructures.h" + + +void clusterLidarWithROI(std::vector &boundingBoxes, std::vector &lidarPoints, float shrinkFactor, cv::Mat &P_rect_xx, cv::Mat &R_rect_xx, cv::Mat &RT); +void clusterKptMatchesWithROI(BoundingBox &boundingBox, std::vector &kptsPrev, std::vector &kptsCurr, std::vector &kptMatches); +void matchBoundingBoxes(std::vector &matches, std::map &bbBestMatches, DataFrame &prevFrame, DataFrame &currFrame); + +void show3DObjects(std::vector &boundingBoxes, cv::Size worldSize, cv::Size imageSize, bool bWait=true); + +void computeTTCCamera(std::vector &kptsPrev, std::vector &kptsCurr, + std::vector kptMatches, double frameRate, double &TTC, cv::Mat *visImg=nullptr); +void computeTTCLidar(std::vector &lidarPointsPrev, + std::vector &lidarPointsCurr, double frameRate, double &TTC); +#endif /* camFusion_hpp */ diff --git a/src/camFusion_Student.cpp b/src/camFusion_Student.cpp new file mode 100644 index 00000000..4e1814da --- /dev/null +++ b/src/camFusion_Student.cpp @@ -0,0 +1,158 @@ + +#include +#include +#include +#include +#include + +#include "camFusion.hpp" +#include "dataStructures.h" + +using namespace std; + + +// Create groups of Lidar points whose projection into the camera falls into the same bounding box +void clusterLidarWithROI(std::vector &boundingBoxes, std::vector &lidarPoints, float shrinkFactor, cv::Mat &P_rect_xx, cv::Mat &R_rect_xx, cv::Mat &RT) +{ + // loop over all Lidar points and associate them to a 2D bounding box + cv::Mat X(4, 1, cv::DataType::type); + cv::Mat Y(3, 1, cv::DataType::type); + + for (auto it1 = lidarPoints.begin(); it1 != lidarPoints.end(); ++it1) + { + // assemble vector for matrix-vector-multiplication + X.at(0, 0) = it1->x; + X.at(1, 0) = it1->y; + X.at(2, 0) = it1->z; + X.at(3, 0) = 1; + + // project Lidar point into camera + Y = P_rect_xx * R_rect_xx * RT * X; + cv::Point pt; + pt.x = Y.at(0, 0) / Y.at(0, 2); // pixel coordinates + pt.y = Y.at(1, 0) / Y.at(0, 2); + + vector::iterator> enclosingBoxes; // pointers to all bounding boxes which enclose the current Lidar point + for (vector::iterator it2 = boundingBoxes.begin(); it2 != boundingBoxes.end(); ++it2) + { + // shrink current bounding box slightly to avoid having too many outlier points around the edges + cv::Rect smallerBox; + smallerBox.x = (*it2).roi.x + shrinkFactor * (*it2).roi.width / 2.0; + smallerBox.y = (*it2).roi.y + shrinkFactor * (*it2).roi.height / 2.0; + smallerBox.width = (*it2).roi.width * (1 - shrinkFactor); + smallerBox.height = (*it2).roi.height * (1 - shrinkFactor); + + // check wether point is within current bounding box + if (smallerBox.contains(pt)) + { + enclosingBoxes.push_back(it2); + } + + } // eof loop over all bounding boxes + + // check wether point has been enclosed by one or by multiple boxes + if (enclosingBoxes.size() == 1) + { + // add Lidar point to bounding box + enclosingBoxes[0]->lidarPoints.push_back(*it1); + } + + } // eof loop over all Lidar points +} + + +void show3DObjects(std::vector &boundingBoxes, cv::Size worldSize, cv::Size imageSize, bool bWait) +{ + // create topview image + cv::Mat topviewImg(imageSize, CV_8UC3, cv::Scalar(255, 255, 255)); + + for(auto it1=boundingBoxes.begin(); it1!=boundingBoxes.end(); ++it1) + { + // create randomized color for current 3D object + cv::RNG rng(it1->boxID); + cv::Scalar currColor = cv::Scalar(rng.uniform(0,150), rng.uniform(0, 150), rng.uniform(0, 150)); + + // plot Lidar points into top view image + int top=1e8, left=1e8, bottom=0.0, right=0.0; + float xwmin=1e8, ywmin=1e8, ywmax=-1e8; + for (auto it2 = it1->lidarPoints.begin(); it2 != it1->lidarPoints.end(); ++it2) + { + // world coordinates + float xw = (*it2).x; // world position in m with x facing forward from sensor + float yw = (*it2).y; // world position in m with y facing left from sensor + xwmin = xwminyw ? ywmax : yw; + + // top-view coordinates + int y = (-xw * imageSize.height / worldSize.height) + imageSize.height; + int x = (-yw * imageSize.width / worldSize.width) + imageSize.width / 2; + + // find enclosing rectangle + top = topy ? bottom : y; + right = right>x ? right : x; + + // draw individual point + cv::circle(topviewImg, cv::Point(x, y), 4, currColor, -1); + } + + // draw enclosing rectangle + cv::rectangle(topviewImg, cv::Point(left, top), cv::Point(right, bottom),cv::Scalar(0,0,0), 2); + + // augment object with some key data + char str1[200], str2[200]; + sprintf(str1, "id=%d, #pts=%d", it1->boxID, (int)it1->lidarPoints.size()); + putText(topviewImg, str1, cv::Point2f(left-250, bottom+50), cv::FONT_ITALIC, 2, currColor); + sprintf(str2, "xmin=%2.2f m, yw=%2.2f m", xwmin, ywmax-ywmin); + putText(topviewImg, str2, cv::Point2f(left-250, bottom+125), cv::FONT_ITALIC, 2, currColor); + } + + // plot distance markers + float lineSpacing = 2.0; // gap between distance markers + int nMarkers = floor(worldSize.height / lineSpacing); + for (size_t i = 0; i < nMarkers; ++i) + { + int y = (-(i * lineSpacing) * imageSize.height / worldSize.height) + imageSize.height; + cv::line(topviewImg, cv::Point(0, y), cv::Point(imageSize.width, y), cv::Scalar(255, 0, 0)); + } + + // display image + string windowName = "3D Objects"; + cv::namedWindow(windowName, 1); + cv::imshow(windowName, topviewImg); + + if(bWait) + { + cv::waitKey(0); // wait for key to be pressed + } +} + + +// associate a given bounding box with the keypoints it contains +void clusterKptMatchesWithROI(BoundingBox &boundingBox, std::vector &kptsPrev, std::vector &kptsCurr, std::vector &kptMatches) +{ + // ... +} + + +// Compute time-to-collision (TTC) based on keypoint correspondences in successive images +void computeTTCCamera(std::vector &kptsPrev, std::vector &kptsCurr, + std::vector kptMatches, double frameRate, double &TTC, cv::Mat *visImg) +{ + // ... +} + + +void computeTTCLidar(std::vector &lidarPointsPrev, + std::vector &lidarPointsCurr, double frameRate, double &TTC) +{ + // ... +} + + +void matchBoundingBoxes(std::vector &matches, std::map &bbBestMatches, DataFrame &prevFrame, DataFrame &currFrame) +{ + // ... +} diff --git a/src/dataStructures.h b/src/dataStructures.h new file mode 100644 index 00000000..72267aef --- /dev/null +++ b/src/dataStructures.h @@ -0,0 +1,40 @@ + +#ifndef dataStructures_h +#define dataStructures_h + +#include +#include +#include + +struct LidarPoint { // single lidar point in space + double x,y,z,r; // x,y,z in [m], r is point reflectivity +}; + +struct BoundingBox { // bounding box around a classified object (contains both 2D and 3D data) + + int boxID; // unique identifier for this bounding box + int trackID; // unique identifier for the track to which this bounding box belongs + + cv::Rect roi; // 2D region-of-interest in image coordinates + int classID; // ID based on class file provided to YOLO framework + double confidence; // classification trust + + std::vector lidarPoints; // Lidar 3D points which project into 2D image roi + std::vector keypoints; // keypoints enclosed by 2D roi + std::vector kptMatches; // keypoint matches enclosed by 2D roi +}; + +struct DataFrame { // represents the available sensor information at the same time instance + + cv::Mat cameraImg; // camera image + + std::vector keypoints; // 2D keypoints within camera image + cv::Mat descriptors; // keypoint descriptors + std::vector kptMatches; // keypoint matches between previous and current frame + std::vector lidarPoints; + + std::vector boundingBoxes; // ROI around detected objects in 2D image coordinates + std::map bbMatches; // bounding box matches between previous and current frame +}; + +#endif /* dataStructures_h */ diff --git a/src/lidarData.cpp b/src/lidarData.cpp new file mode 100644 index 00000000..d540cb88 --- /dev/null +++ b/src/lidarData.cpp @@ -0,0 +1,148 @@ + +#include +#include +#include +#include +#include "lidarData.hpp" + + +using namespace std; + +// remove Lidar points based on min. and max distance in X, Y and Z +void cropLidarPoints(std::vector &lidarPoints, float minX, float maxX, float maxY, float minZ, float maxZ, float minR) +{ + std::vector newLidarPts; + for(auto it=lidarPoints.begin(); it!=lidarPoints.end(); ++it) { + + if( (*it).x>=minX && (*it).x<=maxX && (*it).z>=minZ && (*it).z<=maxZ && (*it).z<=0.0 && abs((*it).y)<=maxY && (*it).r>=minR ) // Check if Lidar point is outside of boundaries + { + newLidarPts.push_back(*it); + } + } + + lidarPoints = newLidarPts; +} + + + +// Load Lidar points from a given location and store them in a vector +void loadLidarFromFile(vector &lidarPoints, string filename) +{ + // allocate 4 MB buffer (only ~130*4*4 KB are needed) + unsigned long num = 1000000; + float *data = (float*)malloc(num*sizeof(float)); + + // pointers + float *px = data+0; + float *py = data+1; + float *pz = data+2; + float *pr = data+3; + + // load point cloud + FILE *stream; + stream = fopen (filename.c_str(),"rb"); + num = fread(data,sizeof(float),num,stream)/4; + + for (int32_t i=0; i &lidarPoints, cv::Size worldSize, cv::Size imageSize, bool bWait) +{ + // create topview image + cv::Mat topviewImg(imageSize, CV_8UC3, cv::Scalar(0, 0, 0)); + + // plot Lidar points into image + for (auto it = lidarPoints.begin(); it != lidarPoints.end(); ++it) + { + float xw = (*it).x; // world position in m with x facing forward from sensor + float yw = (*it).y; // world position in m with y facing left from sensor + + int y = (-xw * imageSize.height / worldSize.height) + imageSize.height; + int x = (-yw * imageSize.height / worldSize.height) + imageSize.width / 2; + + cv::circle(topviewImg, cv::Point(x, y), 5, cv::Scalar(0, 0, 255), -1); + } + + // plot distance markers + float lineSpacing = 2.0; // gap between distance markers + int nMarkers = floor(worldSize.height / lineSpacing); + for (size_t i = 0; i < nMarkers; ++i) + { + int y = (-(i * lineSpacing) * imageSize.height / worldSize.height) + imageSize.height; + cv::line(topviewImg, cv::Point(0, y), cv::Point(imageSize.width, y), cv::Scalar(255, 0, 0)); + } + + // display image + string windowName = "Top-View Perspective of LiDAR data"; + cv::namedWindow(windowName, 2); + cv::imshow(windowName, topviewImg); + if(bWait) + { + cv::waitKey(0); // wait for key to be pressed + } +} + +void showLidarImgOverlay(cv::Mat &img, std::vector &lidarPoints, cv::Mat &P_rect_xx, cv::Mat &R_rect_xx, cv::Mat &RT, cv::Mat *extVisImg) +{ + // init image for visualization + cv::Mat visImg; + if(extVisImg==nullptr) + { + visImg = img.clone(); + } else + { + visImg = *extVisImg; + } + + cv::Mat overlay = visImg.clone(); + + // find max. x-value + double maxVal = 0.0; + for(auto it=lidarPoints.begin(); it!=lidarPoints.end(); ++it) + { + maxVal = maxValx ? it->x : maxVal; + } + + cv::Mat X(4,1,cv::DataType::type); + cv::Mat Y(3,1,cv::DataType::type); + for(auto it=lidarPoints.begin(); it!=lidarPoints.end(); ++it) { + + X.at(0, 0) = it->x; + X.at(1, 0) = it->y; + X.at(2, 0) = it->z; + X.at(3, 0) = 1; + + Y = P_rect_xx * R_rect_xx * RT * X; + cv::Point pt; + pt.x = Y.at(0, 0) / Y.at(0, 2); + pt.y = Y.at(1, 0) / Y.at(0, 2); + + float val = it->x; + int red = min(255, (int)(255 * abs((val - maxVal) / maxVal))); + int green = min(255, (int)(255 * (1 - abs((val - maxVal) / maxVal)))); + cv::circle(overlay, pt, 5, cv::Scalar(0, green, red), -1); + } + + float opacity = 0.6; + cv::addWeighted(overlay, opacity, visImg, 1 - opacity, 0, visImg); + + // return augmented image or wait if no image has been provided + if (extVisImg == nullptr) + { + string windowName = "LiDAR data on image overlay"; + cv::namedWindow( windowName, 3 ); + cv::imshow( windowName, visImg ); + cv::waitKey(0); // wait for key to be pressed + } + else + { + extVisImg = &visImg; + } +} \ No newline at end of file diff --git a/src/lidarData.hpp b/src/lidarData.hpp new file mode 100644 index 00000000..e94e8676 --- /dev/null +++ b/src/lidarData.hpp @@ -0,0 +1,16 @@ + +#ifndef lidarData_hpp +#define lidarData_hpp + +#include +#include +#include + +#include "dataStructures.h" + +void cropLidarPoints(std::vector &lidarPoints, float minX, float maxX, float maxY, float minZ, float maxZ, float minR); +void loadLidarFromFile(std::vector &lidarPoints, std::string filename); + +void showLidarTopview(std::vector &lidarPoints, cv::Size worldSize, cv::Size imageSize, bool bWait=true); +void showLidarImgOverlay(cv::Mat &img, std::vector &lidarPoints, cv::Mat &P_rect_xx, cv::Mat &R_rect_xx, cv::Mat &RT, cv::Mat *extVisImg=nullptr); +#endif /* lidarData_hpp */ diff --git a/src/matching2D.hpp b/src/matching2D.hpp new file mode 100644 index 00000000..5007a6fa --- /dev/null +++ b/src/matching2D.hpp @@ -0,0 +1,30 @@ + +#ifndef matching2D_hpp +#define matching2D_hpp + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "dataStructures.h" + + +void detKeypointsHarris(std::vector &keypoints, cv::Mat &img, bool bVis=false); +void detKeypointsShiTomasi(std::vector &keypoints, cv::Mat &img, bool bVis=false); +void detKeypointsModern(std::vector &keypoints, cv::Mat &img, std::string detectorType, bool bVis=false); +void descKeypoints(std::vector &keypoints, cv::Mat &img, cv::Mat &descriptors, std::string descriptorType); +void matchDescriptors(std::vector &kPtsSource, std::vector &kPtsRef, cv::Mat &descSource, cv::Mat &descRef, + std::vector &matches, std::string descriptorType, std::string matcherType, std::string selectorType); + +#endif /* matching2D_hpp */ diff --git a/src/matching2D_Student.cpp b/src/matching2D_Student.cpp new file mode 100644 index 00000000..6103686a --- /dev/null +++ b/src/matching2D_Student.cpp @@ -0,0 +1,104 @@ + +#include +#include "matching2D.hpp" + +using namespace std; + +// Find best matches for keypoints in two camera images based on several matching methods +void matchDescriptors(std::vector &kPtsSource, std::vector &kPtsRef, cv::Mat &descSource, cv::Mat &descRef, + std::vector &matches, std::string descriptorType, std::string matcherType, std::string selectorType) +{ + // configure matcher + bool crossCheck = false; + cv::Ptr matcher; + + if (matcherType.compare("MAT_BF") == 0) + { + int normType = cv::NORM_HAMMING; + matcher = cv::BFMatcher::create(normType, crossCheck); + } + else if (matcherType.compare("MAT_FLANN") == 0) + { + // ... + } + + // perform matching task + if (selectorType.compare("SEL_NN") == 0) + { // nearest neighbor (best match) + + matcher->match(descSource, descRef, matches); // Finds the best match for each descriptor in desc1 + } + else if (selectorType.compare("SEL_KNN") == 0) + { // k nearest neighbors (k=2) + + // ... + } +} + +// Use one of several types of state-of-art descriptors to uniquely identify keypoints +void descKeypoints(vector &keypoints, cv::Mat &img, cv::Mat &descriptors, string descriptorType) +{ + // select appropriate descriptor + cv::Ptr extractor; + if (descriptorType.compare("BRISK") == 0) + { + + int threshold = 30; // FAST/AGAST detection threshold score. + int octaves = 3; // detection octaves (use 0 to do single scale) + float patternScale = 1.0f; // apply this scale to the pattern used for sampling the neighbourhood of a keypoint. + + extractor = cv::BRISK::create(threshold, octaves, patternScale); + } + else + { + + //... + } + + // perform feature description + double t = (double)cv::getTickCount(); + extractor->compute(img, keypoints, descriptors); + t = ((double)cv::getTickCount() - t) / cv::getTickFrequency(); + cout << descriptorType << " descriptor extraction in " << 1000 * t / 1.0 << " ms" << endl; +} + +// Detect keypoints in image using the traditional Shi-Thomasi detector +void detKeypointsShiTomasi(vector &keypoints, cv::Mat &img, bool bVis) +{ + // compute detector parameters based on image size + int blockSize = 4; // size of an average block for computing a derivative covariation matrix over each pixel neighborhood + double maxOverlap = 0.0; // max. permissible overlap between two features in % + double minDistance = (1.0 - maxOverlap) * blockSize; + int maxCorners = img.rows * img.cols / max(1.0, minDistance); // max. num. of keypoints + + double qualityLevel = 0.01; // minimal accepted quality of image corners + double k = 0.04; + + // Apply corner detection + double t = (double)cv::getTickCount(); + vector corners; + cv::goodFeaturesToTrack(img, corners, maxCorners, qualityLevel, minDistance, cv::Mat(), blockSize, false, k); + + // add corners to result vector + for (auto it = corners.begin(); it != corners.end(); ++it) + { + + cv::KeyPoint newKeyPoint; + newKeyPoint.pt = cv::Point2f((*it).x, (*it).y); + newKeyPoint.size = blockSize; + keypoints.push_back(newKeyPoint); + } + t = ((double)cv::getTickCount() - t) / cv::getTickFrequency(); + cout << "Shi-Tomasi detection with n=" << keypoints.size() << " keypoints in " << 1000 * t / 1.0 << " ms" << endl; + + // visualize results + if (bVis) + { + cv::Mat visImage = img.clone(); + cv::drawKeypoints(img, keypoints, visImage, cv::Scalar::all(-1), cv::DrawMatchesFlags::DRAW_RICH_KEYPOINTS); + string windowName = "Shi-Tomasi Corner Detector Results"; + cv::namedWindow(windowName, 6); + imshow(windowName, visImage); + cv::waitKey(0); + } +} \ No newline at end of file diff --git a/src/objectDetection2D.cpp b/src/objectDetection2D.cpp new file mode 100644 index 00000000..a5f43853 --- /dev/null +++ b/src/objectDetection2D.cpp @@ -0,0 +1,129 @@ + +#include +#include +#include + +#include +#include +#include + +#include "objectDetection2D.hpp" + + +using namespace std; + +// detects objects in an image using the YOLO library and a set of pre-trained objects from the COCO database; +// a set of 80 classes is listed in "coco.names" and pre-trained weights are stored in "yolov3.weights" +void detectObjects(cv::Mat& img, std::vector& bBoxes, float confThreshold, float nmsThreshold, + std::string basePath, std::string classesFile, std::string modelConfiguration, std::string modelWeights, bool bVis) +{ + // load class names from file + vector classes; + ifstream ifs(classesFile.c_str()); + string line; + while (getline(ifs, line)) classes.push_back(line); + + // load neural network + cv::dnn::Net net = cv::dnn::readNetFromDarknet(modelConfiguration, modelWeights); + net.setPreferableBackend(cv::dnn::DNN_BACKEND_OPENCV); + net.setPreferableTarget(cv::dnn::DNN_TARGET_CPU); + + // generate 4D blob from input image + cv::Mat blob; + vector netOutput; + double scalefactor = 1/255.0; + cv::Size size = cv::Size(416, 416); + cv::Scalar mean = cv::Scalar(0,0,0); + bool swapRB = false; + bool crop = false; + cv::dnn::blobFromImage(img, blob, scalefactor, size, mean, swapRB, crop); + + // Get names of output layers + vector names; + vector outLayers = net.getUnconnectedOutLayers(); // get indices of output layers, i.e. layers with unconnected outputs + vector layersNames = net.getLayerNames(); // get names of all layers in the network + + names.resize(outLayers.size()); + for (size_t i = 0; i < outLayers.size(); ++i) // Get the names of the output layers in names + names[i] = layersNames[outLayers[i] - 1]; + + // invoke forward propagation through network + net.setInput(blob); + net.forward(netOutput, names); + + // Scan through all bounding boxes and keep only the ones with high confidence + vector classIds; vector confidences; vector boxes; + for (size_t i = 0; i < netOutput.size(); ++i) + { + float* data = (float*)netOutput[i].data; + for (int j = 0; j < netOutput[i].rows; ++j, data += netOutput[i].cols) + { + cv::Mat scores = netOutput[i].row(j).colRange(5, netOutput[i].cols); + cv::Point classId; + double confidence; + + // Get the value and location of the maximum score + cv::minMaxLoc(scores, 0, &confidence, 0, &classId); + if (confidence > confThreshold) + { + cv::Rect box; int cx, cy; + cx = (int)(data[0] * img.cols); + cy = (int)(data[1] * img.rows); + box.width = (int)(data[2] * img.cols); + box.height = (int)(data[3] * img.rows); + box.x = cx - box.width/2; // left + box.y = cy - box.height/2; // top + + boxes.push_back(box); + classIds.push_back(classId.x); + confidences.push_back((float)confidence); + } + } + } + + // perform non-maxima suppression + vector indices; + cv::dnn::NMSBoxes(boxes, confidences, confThreshold, nmsThreshold, indices); + for(auto it=indices.begin(); it!=indices.end(); ++it) { + + BoundingBox bBox; + bBox.roi = boxes[*it]; + bBox.classID = classIds[*it]; + bBox.confidence = confidences[*it]; + bBox.boxID = (int)bBoxes.size(); // zero-based unique identifier for this bounding box + + bBoxes.push_back(bBox); + } + + // show results + if(bVis) { + + cv::Mat visImg = img.clone(); + for(auto it=bBoxes.begin(); it!=bBoxes.end(); ++it) { + + // Draw rectangle displaying the bounding box + int top, left, width, height; + top = (*it).roi.y; + left = (*it).roi.x; + width = (*it).roi.width; + height = (*it).roi.height; + cv::rectangle(visImg, cv::Point(left, top), cv::Point(left+width, top+height),cv::Scalar(0, 255, 0), 2); + + string label = cv::format("%.2f", (*it).confidence); + label = classes[((*it).classID)] + ":" + label; + + // Display label at the top of the bounding box + int baseLine; + cv::Size labelSize = getTextSize(label, cv::FONT_ITALIC, 0.5, 1, &baseLine); + top = max(top, labelSize.height); + rectangle(visImg, cv::Point(left, top - round(1.5*labelSize.height)), cv::Point(left + round(1.5*labelSize.width), top + baseLine), cv::Scalar(255, 255, 255), cv::FILLED); + cv::putText(visImg, label, cv::Point(left, top), cv::FONT_ITALIC, 0.75, cv::Scalar(0,0,0),1); + + } + + string windowName = "Object classification"; + cv::namedWindow( windowName, 1 ); + cv::imshow( windowName, visImg ); + cv::waitKey(0); // wait for key to be pressed + } +} diff --git a/src/objectDetection2D.hpp b/src/objectDetection2D.hpp new file mode 100644 index 00000000..b820ac6b --- /dev/null +++ b/src/objectDetection2D.hpp @@ -0,0 +1,13 @@ + +#ifndef objectDetection2D_hpp +#define objectDetection2D_hpp + +#include +#include + +#include "dataStructures.h" + +void detectObjects(cv::Mat& img, std::vector& bBoxes, float confThreshold, float nmsThreshold, + std::string basePath, std::string classesFile, std::string modelConfiguration, std::string modelWeights, bool bVis); + +#endif /* objectDetection2D_hpp */