added initial code

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Niki Amini-Naieni
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,30 @@
+# 💥Instant Uncertainty Calibration of NeRFs Using a Meta-Calibrator (ECCV 2024)
+
+Niki Amini-Naieni, Tomas Jakab, Andrea Vedaldi, Ronald Clark
+
+Official PyTorch implementation for Instant Uncertainty Calibration of NeRFs Using a Meta-Calibrator. Details can be found in the paper, [[Paper]](https://arxiv.org/abs/2312.02350) [[Project page]](https://niki-amini-naieni.github.io/instantcalibration.github.io/).
+
+<img src=img/teaser.png width="50%"/>
+
+## Instructions
+
+- ```get_metacal_data.py``` provides code to get training data for meta-calibrator
+- ```extract_features.py``` provides code to extract DINOv2 features from inferred images and uncalibrated uncertainty maps
+- ```train_calibrator_all_scenes.py``` provides code to train meta-calibrator on calibration curves from training scenes and predict the calibration curve of a test scene
+- ```get_final_metrics.py``` provides code to evaluate the uncertainties and image quality of the uncalibrated and calibrated NeRF
+
+## Citation
+
+```
+@inproceedings{AminiNaieni24,
+    title={Instant Uncertainty Calibration of {NeRFs} Using a Meta-Calibrator},
+    author={Niki Amini-Naieni and Tomas Jakab and Andrea Vedaldi and Ronald Clark},
+    booktitle={ECCV},
+    year={2024}
+}
+```
+
+### Acknowledgements
+
+This repository uses code from the [FlipNeRF repository](https://github.com/shawn615/FlipNeRF). 
+

configs/blender4.gin

@@ -0,0 +1,45 @@
+# Blender settings
+Config.data_dir = "data/nerf_synthetic"
+Config.checkpoint_dir = "out/flipnerf_blender4_lego"
+Config.dataset_loader = 'blender'
+Config.blender_scene = 'lego'
+Config.white_background = True
+Config.factor = 2
+Config.render_factor = 2
+MLP.density_noise = 1.
+
+Config.n_input_views = 4
+Config.max_steps = 78125
+
+Config.lr_init = 0.001
+Config.lr_final = 0.00001
+Config.lr_delay_steps = 512
+Config.lr_delay_mult = 0.01
+Config.grad_max_norm = 0.1
+Config.grad_max_val = 0.1
+
+Config.anneal_nearfar = True
+Config.anneal_nearfar_steps = 256
+Config.anneal_nearfar_perc = 0.5
+
+############## flipnerf ##############
+Config.nll_loss = True
+Config.nll_loss_maxstep = 512
+Config.nll_loss_weight_start = 4.0
+Config.nll_loss_weight_end = 1e-3
+Config.flip_nll_loss_maxstep = 512
+Config.flip_nll_loss_weight_start = 0.4
+Config.flip_nll_loss_weight_end = 1e-4
+Config.ue_loss = True
+Config.ue_loss_maxstep = 512
+Config.ue_loss_weight_start = 1e-4
+Config.ue_loss_weight_end = 1e-1
+Config.flip_ue_loss_weight = 1e-2
+Config.bfc_loss = True
+Config.bfc_loss_weight = 0.1
+Config.ori_loss = True
+Config.ori_loss_weight = 0.1
+############## flipnerf ##############
+
+Config.train_render_every = 5000
+Config.load_random_rays = False

configs/blender8.gin

@@ -0,0 +1,45 @@
+# Blender settings
+Config.data_dir = "data/nerf_synthetic"
+Config.checkpoint_dir = "out/flipnerf_blender8_ship"
+Config.dataset_loader = 'blender'
+Config.blender_scene = 'ship'
+Config.white_background = True
+Config.factor = 2
+Config.render_factor = 2
+MLP.density_noise = 1.
+
+Config.n_input_views = 8
+Config.max_steps = 156250
+
+Config.lr_init = 0.001
+Config.lr_final = 0.00001
+Config.lr_delay_steps = 1024
+Config.lr_delay_mult = 0.01
+Config.grad_max_norm = 0.1
+Config.grad_max_val = 0.1
+
+Config.anneal_nearfar = True
+Config.anneal_nearfar_steps = 256
+Config.anneal_nearfar_perc = 0.5
+
+############## flipnerf ##############
+Config.nll_loss = True
+Config.nll_loss_maxstep = 512
+Config.nll_loss_weight_start = 4.0
+Config.nll_loss_weight_end = 1e-3
+Config.flip_nll_loss_maxstep = 512
+Config.flip_nll_loss_weight_start = 0.04
+Config.flip_nll_loss_weight_end = 1e-5
+Config.ue_loss = True
+Config.ue_loss_maxstep = 512
+Config.ue_loss_weight_start = 1e-5
+Config.ue_loss_weight_end = 1e-2
+Config.flip_ue_loss_weight = 1e-3
+Config.bfc_loss = True
+Config.bfc_loss_weight = 0.01
+Config.ori_loss = True
+Config.ori_loss_weight = 0.01
+############## flipnerf ##############
+
+Config.train_render_every = 5000
+Config.load_random_rays = False

configs/dtu3.gin

@@ -0,0 +1,48 @@
+# DTU settings
+Config.data_dir = "/mnt/data0/niki/data/Rectified/"
+Config.dtu_mask_path = '/mnt/data0/niki/data/submission_data/idrmasks'
+Config.checkpoint_dir = "checkpoints/dtu3/scan30"
+Config.dtu_scan = "scan30"
+Config.dataset_loader = 'dtu'
+Config.white_background = False
+Config.near = 0.5
+Config.far = 3.5
+Config.factor = 4
+Config.render_factor = 4
+MLP.density_noise = 1.
+
+Config.n_input_views = 3
+Config.max_steps = 43945
+
+Config.lr_init = 0.002
+Config.lr_final = 0.00002
+Config.lr_delay_steps = 512
+Config.lr_delay_mult = 0.01
+Config.grad_max_norm = 0.1
+Config.grad_max_val = 0.1
+
+Config.anneal_nearfar = True
+Config.anneal_nearfar_steps = 256
+Config.anneal_nearfar_perc = 0.5
+
+############## flipnerf ##############
+Config.nll_loss = True
+Config.nll_loss_maxstep = 512
+Config.nll_loss_weight_start = 4.0
+Config.nll_loss_weight_end = 1e-3
+Config.flip_nll_loss_maxstep = 512
+Config.flip_nll_loss_weight_start = 0.4
+Config.flip_nll_loss_weight_end = 1e-4
+Config.ue_loss = True
+Config.ue_loss_maxstep = 512
+Config.ue_loss_weight_start = 1e-4
+Config.ue_loss_weight_end = 1e-1
+Config.flip_ue_loss_weight = 1e-3
+Config.bfc_loss = True
+Config.bfc_loss_weight = 0.1
+Config.ori_loss = True
+Config.ori_loss_weight = 0.1
+############## flipnerf ##############
+
+Config.train_render_every = 5000
+Config.load_random_rays = False

configs/dtu6.gin

@@ -0,0 +1,48 @@
+# DTU settings
+Config.data_dir = "data/DTU/Rectified/"
+Config.dtu_mask_path = 'data/submission_data/idrmasks'
+Config.checkpoint_dir = "out/flipnerf_dtu6_scan21"
+Config.dtu_scan = "scan21"
+Config.dataset_loader = 'dtu'
+Config.white_background = False
+Config.near = 0.5
+Config.far = 3.5
+Config.factor = 4
+Config.render_factor = 4
+MLP.density_noise = 1.
+
+Config.n_input_views = 6
+Config.max_steps = 87891
+
+Config.lr_init = 0.002
+Config.lr_final = 0.00002
+Config.lr_delay_steps = 1024
+Config.lr_delay_mult = 0.01
+Config.grad_max_norm = 0.1
+Config.grad_max_val = 0.1
+
+Config.anneal_nearfar = True
+Config.anneal_nearfar_steps = 256
+Config.anneal_nearfar_perc = 0.5
+
+############## flipnerf ##############
+Config.nll_loss = True
+Config.nll_loss_maxstep = 512
+Config.nll_loss_weight_start = 4.0
+Config.nll_loss_weight_end = 1e-3
+Config.flip_nll_loss_maxstep = 512
+Config.flip_nll_loss_weight_start = 0.04
+Config.flip_nll_loss_weight_end = 1e-5
+Config.ue_loss = True
+Config.ue_loss_maxstep = 512
+Config.ue_loss_weight_start = 1e-5
+Config.ue_loss_weight_end = 1e-2
+Config.flip_ue_loss_weight = 1e-4
+Config.bfc_loss = True
+Config.bfc_loss_weight = 0.01
+Config.ori_loss = True
+Config.ori_loss_weight = 0.01
+############## flipnerf ##############
+
+Config.train_render_every = 5000
+Config.load_random_rays = False

configs/dtu9.gin

@@ -0,0 +1,48 @@
+# DTU settings
+Config.data_dir = "data/DTU/Rectified/"
+Config.dtu_mask_path = 'data/submission_data/idrmasks'
+Config.checkpoint_dir = "out/flipnerf_dtu9_scan30"
+Config.dtu_scan = "scan30"
+Config.dataset_loader = 'dtu'
+Config.white_background = False
+Config.near = 0.5
+Config.far = 3.5
+Config.factor = 4
+Config.render_factor = 4
+MLP.density_noise = 1.
+
+Config.n_input_views = 9
+Config.max_steps = 131836
+
+Config.lr_init = 0.002
+Config.lr_final = 0.00002
+Config.lr_delay_steps = 1024
+Config.lr_delay_mult = 0.01
+Config.grad_max_norm = 0.1
+Config.grad_max_val = 0.1
+
+Config.anneal_nearfar = False
+Config.anneal_nearfar_steps = 256
+Config.anneal_nearfar_perc = 0.5
+
+############## flipnerf ##############
+Config.nll_loss = True
+Config.nll_loss_maxstep = 512
+Config.nll_loss_weight_start = 4.0
+Config.nll_loss_weight_end = 1e-3
+Config.flip_nll_loss_maxstep = 512
+Config.flip_nll_loss_weight_start = 0.004
+Config.flip_nll_loss_weight_end = 1e-6
+Config.ue_loss = True
+Config.ue_loss_maxstep = 512
+Config.ue_loss_weight_start = 1e-6
+Config.ue_loss_weight_end = 1e-3
+Config.flip_ue_loss_weight = 1e-5
+Config.bfc_loss = True
+Config.bfc_loss_weight = 0.001
+Config.ori_loss = True
+Config.ori_loss_weight = 0.001
+############## flipnerf ##############
+
+Config.train_render_every = 5000
+Config.load_random_rays = False

configs/llff_fern.gin

@@ -0,0 +1,50 @@
+# LLFF settings
+Config.data_dir = "/mnt/data0/niki/data/nerf_llff_data"
+Config.checkpoint_dir = "checkpoints/llff3/fern"
+Config.dataset_loader = 'llff'
+Config.llff_scan = 'fern'
+Config.white_background = False
+Config.factor = 8
+Config.render_factor = 8
+Config.near = 0
+Config.far = 1
+MipNerfModel.ray_shape = 'cylinder'
+MLP.density_noise = 1.
+
+Config.n_input_views = 3
+Config.max_steps = 69768
+
+Config.lr_init = 0.002
+Config.lr_final = 0.00002
+Config.lr_delay_steps = 512
+Config.lr_delay_mult = 0.01
+Config.grad_max_norm = 0.1
+Config.grad_max_val = 0.1
+
+Config.anneal_nearfar = True
+Config.anneal_nearfar_steps = 512
+Config.anneal_nearfar_perc = 0.0001
+Config.anneal_mid_perc = 1.0
+
+############## flipnerf ##############
+Config.nll_loss = True
+Config.nll_loss_maxstep = 512
+Config.nll_loss_weight_start = 4.0
+Config.nll_loss_weight_end = 1e-3
+Config.flip_nll_loss_maxstep = 512
+Config.flip_nll_loss_weight_start = 0.004
+Config.flip_nll_loss_weight_end = 1e-6
+Config.ue_loss = True
+Config.ue_loss_maxstep = 512
+Config.ue_loss_weight_start = 1e-6
+Config.ue_loss_weight_end = 1e-3
+Config.flip_ue_loss_weight = 1e-5
+Config.bfc_loss = True
+Config.bfc_loss_weight = 0.001
+Config.ori_loss = True
+Config.ori_loss_weight = 0.001
+Config.llff_flip_mask = True
+############## flipnerf ##############
+
+Config.train_render_every = 5000
+Config.load_random_rays = False

configs/llff_flower.gin

@@ -0,0 +1,50 @@
+# LLFF settings
+Config.data_dir = "/mnt/data0/niki/data/nerf_llff_data"
+Config.checkpoint_dir = "checkpoints/llff3/flower"
+Config.dataset_loader = 'llff'
+Config.llff_scan = 'flower'
+Config.white_background = False
+Config.factor = 8
+Config.render_factor = 8
+Config.near = 0
+Config.far = 1
+MipNerfModel.ray_shape = 'cylinder'
+MLP.density_noise = 1.
+
+Config.n_input_views = 3
+Config.max_steps = 69768
+
+Config.lr_init = 0.002
+Config.lr_final = 0.00002
+Config.lr_delay_steps = 512
+Config.lr_delay_mult = 0.01
+Config.grad_max_norm = 0.1
+Config.grad_max_val = 0.1
+
+Config.anneal_nearfar = True
+Config.anneal_nearfar_steps = 512
+Config.anneal_nearfar_perc = 0.0001
+Config.anneal_mid_perc = 1.0
+
+############## flipnerf ##############
+Config.nll_loss = True
+Config.nll_loss_maxstep = 512
+Config.nll_loss_weight_start = 4.0
+Config.nll_loss_weight_end = 1e-3
+Config.flip_nll_loss_maxstep = 512
+Config.flip_nll_loss_weight_start = 0.004
+Config.flip_nll_loss_weight_end = 1e-6
+Config.ue_loss = True
+Config.ue_loss_maxstep = 512
+Config.ue_loss_weight_start = 1e-6
+Config.ue_loss_weight_end = 1e-3
+Config.flip_ue_loss_weight = 1e-5
+Config.bfc_loss = True
+Config.bfc_loss_weight = 0.001
+Config.ori_loss = True
+Config.ori_loss_weight = 0.001
+Config.llff_flip_mask = True
+############## flipnerf ##############
+
+Config.train_render_every = 5000
+Config.load_random_rays = False