Add files via upload

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Saugat Malla
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -1 +1,34 @@
-# AR-Calibration
+# AR-Calibration
+
+## Overview:
+- Calibrate a camera and then use the calibration to generate virtual objects in a scene.
+- The program detects a target and then place a virtual object in the scene relative to the target that moves and orients itself correctly given motion of the camera or target. 
+
+## Implementation:
+- Detect and Extract Target Corners:
+    - System that detects a target and extracts target corners.    
+- Select Calibration Images:
+    - Image used for calibration.
+    - Save the corner locations and the correcponding 3D world points.
+    - If user key presses 's', then we store the vector of corners found by the last successful target detection into a corner_list.
+    - Chessboard corners highlighted.
+- Calibrate the Camera:
+    - Calibration is continuously updated each time the user adds a new calibration beyond some minium number.
+    - Get the current per pixel error after each calibration.
+    - Error estimation.
+    Calibration matirix along with corresponding re-projection error.
+- Calculate current position of the camera:
+    - Read the camera calibration parameters from a file and then start a video loop.
+    - For each frame, it tries to detect a target.
+    - If found, it grabs the locations of the corners, and then uses [solvePNP](https://docs.opencv.org/3.4/d9/d0c/group__calib3d.html#ga549c2075fac14829ff4a58bc931c033d) to get the board's pose (rotation and translation).
+- Project outside corners or 3D Axes:
+    - Using [projectPoints](https://docs.opencv.org/3.4/d9/d0c/group__calib3d.html#ga1019495a2c8d1743ed5cc23fa0daff8c) function to project the 3D points corresponding to at least four corners of the target onto the image plane in real time as the target or camera moves around.
+    - 3D axes on the target attached to the origin which will help build virtual object.
+- Creating virtual object
+    - Construct a virtual object in 3D world space made out of lines that floats above the board.
+    - Project the virtual object to the image and draw the lines in the image whie the object stays in the correct orientation as the camera moves around.
+- Detect Robust Features 
+    - Harris corners and SURF features.
+    - Program that shows where the features are in the image in a video stream.
+    - Generate a pattern and display where the features show up in the pattern.
+    - Different thesholds and setting are used to for features.

calibration_params.txt

@@ -0,0 +1,11 @@
+Camera matrix:
+2836.29 0 678.176 
+0 2756.2 -69.3719 
+0 0 1 
+Distortion coefficients:
+0.130029 
+-1.16806 
+-0.00198507 
+-0.0186432 
+2.48141 
+Reprojection error:2.06456

cameraCalibration.cpp

@@ -0,0 +1,157 @@
+/*
+    Saugat Malla
+    Project 4
+*/
+
+/*
+    Code for taks 1-3 
+*/
+
+
+#include<iostream> // Header file for input and output operations
+#include<opencv2/opencv.hpp> // OpenCV header file
+#include<vector> // Header file for vector container
+#include<fstream> // Header file for file stream operations
+
+using namespace std;
+
+int main(){
+
+    // The dimensions of the chessboard (inner corners)
+    cv::Size chessboardSize(9,6);
+
+    // Video Capture
+    cv::VideoCapture cap(0); // Initialize video capture from default camera (index 0)
+    if(!cap.isOpened()){ // Check if camera is opened successfully
+        cerr<<"Error: Unable to open camera"<<endl; // Display error message
+        return -1; // Return error code
+    }
+
+    // Store detected corner locations for each image
+    vector<vector<cv::Point2f>> cornerList;
+
+    // Store 3D world positions of corners
+    vector<cv::Vec3f> pointSet;
+
+    // Store point sets for multiple images
+    vector<vector<cv::Vec3f>> pointList;
+
+    // Vector to store calibration images
+    vector<cv::Mat> calibrationImages;
+
+    while(true){
+
+        // Capturing the frames
+        cv::Mat frame;
+        cap>>frame; // Read a frame from the camera
+
+        // Convert frame to grayscale
+        cv::Mat gray;
+        cv::cvtColor(frame, gray, cv::COLOR_BGR2GRAY); // Convert frame to grayscale
+
+        // Find chessboard corners
+        vector<cv::Point2f> corners;
+        bool patternFound = cv::findChessboardCorners(gray, chessboardSize, corners); // Find chessboard corners in the grayscale image
+
+        if(patternFound){
+
+            // Refine corner locations
+            cv::cornerSubPix(gray, corners, cv::Size(11,11), cv::Size(-1,-1), cv::TermCriteria(cv::TermCriteria::EPS + cv::TermCriteria::MAX_ITER, 30, 0.001)); // Refine corner locations using sub-pixel accuracy
+
+            // Draw corners on the image
+            cv::drawChessboardCorners(frame, chessboardSize, cv::Mat(corners), patternFound); // Draw detected chessboard corners on the frame
+
+            // Save corner locations and corresponding world points when 's' is pressed
+            char key = cv::waitKey(0); // Wait for key press
+            if(key=='s'){
+                cornerList.push_back(corners); // Store detected corner locations
+
+                // Generate 3D world points for the corners
+                for(int i=0;i<chessboardSize.height;++i){
+                    for(int j=0;j<chessboardSize.width;++j){
+                        pointSet.push_back(cv::Vec3f(j,-i,0)); // Assuming Z=0 for all corners
+                    }
+                }
+
+                pointList.push_back(pointSet); // Store 3D world points corresponding to corners
+                calibrationImages.push_back(frame.clone()); // Store the calibration image
+
+                // Clear the pointSet for the next calibration image
+                pointSet.clear();
+
+                // Print the number of corners and coordinates of the first corner
+                cout<<"Number of corners:"<<corners.size()<<endl;
+                if(!corners.empty()){
+                    cout<<"Coordinates of the first corner: ("<<corners[0].x<<", "<<corners[0].y<<")"<<endl;
+                }
+            }
+        }
+
+        cv::imshow("frame", frame); // Display the frame
+
+        // Break the loop if 'q' is pressed
+        if(cv::waitKey(1) == 'q'){
+            break; // Exit the loop
+        }
+    }
+
+    // Release VideoCapture
+    cap.release();
+
+    // Check if enough calibration frames are selected
+    if(cornerList.size() < 5){
+        cerr<<"Error: Not enough calibration frames selected"<<endl; // Display error message
+        return -1; // Return error code
+    }
+
+    // Calibrate the camera
+    cv::Mat cameraMatrix = cv::Mat::eye(3,3, CV_64F); // Initialize camera matrix as identity matrix
+    cv::Mat distCoeffs = cv::Mat::zeros(8,1, CV_64F); // Initialize distortion coefficients as zeros
+    vector<cv::Mat> rvecs, tvecs; // Vectors to store rotation and translation vectors
+
+    double rms = cv::calibrateCamera(pointList, cornerList, chessboardSize, cameraMatrix, distCoeffs, rvecs, tvecs); // Calibrate the camera
+
+    // Print camera matrix and distortion coefficients
+    cout<<"Camera matrix: "<< cameraMatrix<<endl;
+    cout<<"Distortion coefficients: "<<distCoeffs<<endl;
+    cout<<"Reprojection error: "<<rms<<endl;
+
+    // Save intrinsic parameters to a file
+    ofstream outFile("calibration_params.txt"); // Open file for writing
+    if (outFile.is_open()) {
+        outFile << "Camera matrix:" << endl;
+        for (int i = 0; i < cameraMatrix.rows; ++i) {
+            for (int j = 0; j < cameraMatrix.cols; ++j) {
+                outFile << cameraMatrix.at<double>(i, j) << " "; // Write camera matrix to file
+            }
+            outFile << endl;
+        }
+
+        outFile << "Distortion coefficients:" << endl;
+        // Save distortion coefficients for the fixed number of times
+        for (int i = 0; i < 5; ++i) {
+            for (int j = 0; j < distCoeffs.cols; ++j) {
+                outFile << distCoeffs.at<double>(i, j) << " "; // Write distortion coefficients to file
+            }
+            outFile << endl;
+        }
+
+        outFile << "Reprojection error:" << rms << endl;
+
+        outFile.close(); // Close the file
+        cout << "Intrinsic parameters saved to 'calibration_params.txt'." << endl;
+    } else {
+        cerr << "Error: Unable to write to file" << endl; // Display error message
+    }
+
+    // Show the calibration images with detected corners (optional)
+    for (size_t i = 0; i < calibrationImages.size(); ++i) {
+        cv::imshow("Calibration Image " + std::to_string(i+1), calibrationImages[i]); // Display each calibration image
+        cv::waitKey(0); // Wait for key press
+    }
+
+    // Close all windows
+    cv::destroyAllWindows();
+
+    return 0; // Return success
+}

featureORB.cpp

@@ -0,0 +1,65 @@
+/*
+    Saugat Malla
+    Project 4
+*/
+
+/*
+    Code for extension 3
+*/
+
+#include <iostream> // Header file for input and output operations
+#include <opencv2/opencv.hpp> // OpenCV header file
+
+using namespace std;
+
+int main() {
+    cv::VideoCapture cap(0); // Initialize video capture from camera
+    if (!cap.isOpened()) { // If camera is not opened
+        cerr << "Error: Unable to open camera" << endl; // Display error message
+        return -1; // Return error code
+    }
+
+    // Load the pattern image
+    cv::Mat pattern = cv::imread("checkerboard.jpg"); // Load pattern image
+    if (pattern.empty()) { // If pattern image is not loaded
+        cerr << "Error: Unable to load pattern image" << endl; // Display error message
+        return -1; // Return error code
+    }
+
+    // Initialize ORB detector
+    cv::Ptr<cv::ORB> orb = cv::ORB::create(); // Create ORB detector
+
+    while (true) { // Infinite loop
+        // Capture frame from camera
+        cv::Mat frame;
+        cap >> frame; // Capture frame from camera
+
+        // Detect ORB features and compute descriptors
+        vector<cv::KeyPoint> keypointsPattern, keypointsFrame; // Initialize keypoints vectors
+        cv::Mat descriptorsPattern, descriptorsFrame; // Initialize descriptors matrices
+        orb->detectAndCompute(pattern, cv::noArray(), keypointsPattern, descriptorsPattern); // Detect features and compute descriptors for pattern image
+        orb->detectAndCompute(frame, cv::noArray(), keypointsFrame, descriptorsFrame); // Detect features and compute descriptors for frame
+
+        // Match descriptors
+        cv::BFMatcher matcher(cv::NORM_HAMMING); // Initialize BFMatcher
+        vector<cv::DMatch> matches; // Initialize matches vector
+        matcher.match(descriptorsPattern, descriptorsFrame, matches); // Match descriptors between pattern and frame
+
+        // Draw matches on frame
+        cv::Mat imgMatches; // Initialize image for matches
+        cv::drawMatches(pattern, keypointsPattern, frame, keypointsFrame, matches, imgMatches); // Draw matches on frame
+
+        // Show the frame with matches
+        cv::imshow("Matches", imgMatches); // Display frame with matches
+
+        // Break the loop if 'q' is pressed
+        if (cv::waitKey(1) == 'q') { // If 'q' key is pressed
+            break; // Exit the loop
+        }
+    }
+
+    cap.release(); // Release the camera
+    cv::destroyAllWindows(); // Close all OpenCV windows
+
+    return 0; // Return success
+}

makefile

@@ -0,0 +1,29 @@
+CXX = g++
+CXXFLAGS = -std=c++11
+LIBS = -L/opt/homebrew/lib -lopencv_imgcodecs -lopencv_imgproc -lopencv_highgui -lopencv_core -lopencv_videoio -lopencv_videostab -lopencv_objdetect -lopencv_dnn -lopencv_calib3d -lopencv_features2d -lopencv_xfeatures2d
+INCLUDES = -I/opt/homebrew/include/opencv4
+
+
+cameraCalibration: cameraCalibration.cpp
+	$(CXX) $(CXXFLAGS) $(INCLUDES) -o $@ $^ $(LIBS)
+
+projection: projection.cpp
+	$(CXX) $(CXXFLAGS) $(INCLUDES) -o $@ $^ $(LIBS)
+
+surf: surf.cpp
+	$(CXX) $(CXXFLAGS) $(INCLUDES) -o $@ $^ $(LIBS)
+
+projectionGrid: projectionGrid.cpp
+	$(CXX) $(CXXFLAGS) $(INCLUDES) -o $@ $^ $(LIBS)
+
+projectionCube: projectionCube.cpp
+	$(CXX) $(CXXFLAGS) $(INCLUDES) -o $@ $^ $(LIBS)
+
+featureORB: featureORB.cpp
+	$(CXX) $(CXXFLAGS) $(INCLUDES) -o $@ $^ $(LIBS)
+
+
+.PHONY: clean
+
+clean:
+	rm -f cameraCalibration projection projectionGrid projectionCube featureORB