comparison

Author: monurcan

benchmark_configs/exp_83_8augcount_newdatasets_finalized_answerlevels_temp_confidence.json

@@ -0,0 +1,62 @@
+{
+    "model": {
+        "TTAugAdapter_SmolVLM2_2B_8_AnswerLevelTemperatureConfidenceSelector": {
+            "class": "TTAugAdapter_SmolVLM2",
+            "model_args": {
+                "model_path": "HuggingFaceTB/SmolVLM2-2.2B-Instruct"
+            },
+            "text_aug_args": {
+                "gpt_paraphraser_strategy": "simple_paraphraser",
+                "save_or_load": null,
+                "path_text_aug_to_file": null
+            },
+            "image_aug_args": {
+                "aug_strength": "high",
+                "save_or_load_generativeimg": null,
+                "strength_gen_aug": 0.25,
+                "path_generativeimg_to_directory": null
+            },
+            "token_selection_aggregation_method": "answer_level_temperature_confidence_scores",
+            "number_of_versions": 8,
+            "save_inputs_for_debugging": false
+        }
+    },
+    "data": {
+        "OCRBench": {
+            "class": "OCRBench",
+            "dataset": "OCRBench"
+        },
+        "AMBER": {
+            "class": "ImageYORNDataset",
+            "dataset": "AMBER"
+        },
+        "TextVQA_VAL": {
+            "class": "ImageVQADataset",
+            "dataset": "TextVQA_VAL"
+        },
+        "AI2D_TEST": {
+            "class": "ImageMCQDataset",
+            "dataset": "AI2D_TEST"
+        },
+        "OCRVQA_TEST": {
+            "class": "ImageVQADataset",
+            "dataset": "OCRVQA_TEST"
+        },
+        "ChartQA_TEST": {
+            "class": "ImageVQADataset",
+            "dataset": "ChartQA_TEST"
+        },
+        "GQA_TestDev_Balanced": {
+            "class": "ImageVQADataset",
+            "dataset": "GQA_TestDev_Balanced"
+        },
+        "COCO_VAL": {
+            "class": "ImageCaptionDataset",
+            "dataset": "COCO_VAL"
+        },
+        "MME-RealWorld-Lite": {
+            "class": "MMERealWorld",
+            "dataset": "MME-RealWorld-Lite"
+        }
+    }
+}

benchmark_configs/exp_83_8augcount_newdatasets_finalized_answerlevels_temp_mllmsynthesizer.json

@@ -0,0 +1,62 @@
+{
+    "model": {
+        "TTAugAdapter_SmolVLM2_2B_8_AnswerLevelTemperatureMLLMSynthesizer": {
+            "class": "TTAugAdapter_SmolVLM2",
+            "model_args": {
+                "model_path": "HuggingFaceTB/SmolVLM2-2.2B-Instruct"
+            },
+            "text_aug_args": {
+                "gpt_paraphraser_strategy": "simple_paraphraser",
+                "save_or_load": null,
+                "path_text_aug_to_file": null
+            },
+            "image_aug_args": {
+                "aug_strength": "high",
+                "save_or_load_generativeimg": null,
+                "strength_gen_aug": 0.25,
+                "path_generativeimg_to_directory": null
+            },
+            "token_selection_aggregation_method": "answer_level_temperature_mllm_synthesizer",
+            "number_of_versions": 8,
+            "save_inputs_for_debugging": false
+        }
+    },
+    "data": {
+        "OCRBench": {
+            "class": "OCRBench",
+            "dataset": "OCRBench"
+        },
+        "AMBER": {
+            "class": "ImageYORNDataset",
+            "dataset": "AMBER"
+        },
+        "TextVQA_VAL": {
+            "class": "ImageVQADataset",
+            "dataset": "TextVQA_VAL"
+        },
+        "AI2D_TEST": {
+            "class": "ImageMCQDataset",
+            "dataset": "AI2D_TEST"
+        },
+        "OCRVQA_TEST": {
+            "class": "ImageVQADataset",
+            "dataset": "OCRVQA_TEST"
+        },
+        "ChartQA_TEST": {
+            "class": "ImageVQADataset",
+            "dataset": "ChartQA_TEST"
+        },
+        "GQA_TestDev_Balanced": {
+            "class": "ImageVQADataset",
+            "dataset": "GQA_TestDev_Balanced"
+        },
+        "COCO_VAL": {
+            "class": "ImageCaptionDataset",
+            "dataset": "COCO_VAL"
+        },
+        "MME-RealWorld-Lite": {
+            "class": "MMERealWorld",
+            "dataset": "MME-RealWorld-Lite"
+        }
+    }
+}

benchmark_results/n_samples_1000/exp_83_8augcount_newdatasets_finalized_answerlevels_temp_confidence/TTAugAdapter_SmolVLM2_2B_8_AnswerLevelTemperatureConfidenceSelector/T20251115_Gfa2ed00a/TTAugAdapter_SmolVLM2_2B_8_AnswerLevelTemperatureConfidenceSelector_AI2D_TEST.xlsx

@@ -0,0 +1,2 @@
+"split","Overall","atomStructure","eclipses","faultsEarthquakes","foodChainsWebs","lifeCycles","moonPhaseEquinox","partsOfA","partsOfTheEarth","photosynthesisRespiration","rockCycle","rockStrata","solarSystem","typesOf","volcano","waterCNPCycle"
+"none","0.691","1.0","0.8181818181818182","0.7","0.7803617571059431","0.6183206106870229","0.5454545454545454","0.6928104575163399","0.6","0.72","0.631578947368421","0.5384615384615384","0.75","0.6379310344827587","1.0","0.3125"