Cross annotation (#170)

Co-authored-by: Shorouq <[email protected]>
VUB-HYDR · Oct 18, 2024 · 5e15e5c · 5e15e5c
1 parent 26823d8
commit 5e15e5c
Show file tree

Hide file tree

Showing 82 changed files with 386 additions and 0 deletions.
diff --git a/Database/evaluation_results/README.md b/Database/evaluation_results/README.md
@@ -12,3 +12,5 @@ setting 5 & 1.0 for location, 0 for others & 0.6 & 1.0 \
 
 Based on the evaluation result of V3 cross these experiments, we select setting 2 for the further processing.
 For the V4 and V5, we only use setting 2 for evaluation.
+
+For double annotation, we have two annotators: one for the gold data and one for the output data. The annotator for the gold data worked prior to the implementation of the L1-3 guideline, while the annotator for the output data followed the new guideline. The L2 and L3 information in the gold data is extracted from the Location field. Information at the country level is assigned to L2, while sub-national level information is assigned to L3.
diff --git a/Database/evaluation_results/double_annotation/test/l1/gold_matched_currencies_l1.parquet b/Database/evaluation_results/double_annotation/test/l1/gold_matched_currencies_l1.parquet
diff --git a/Database/evaluation_results/double_annotation/test/l1/l1_all_10_avg_results.json b/Database/evaluation_results/double_annotation/test/l1/l1_all_10_avg_results.json
@@ -0,0 +1,36 @@
+{
+  "Coverage": 0.0,
+  "Weighted_Score": 0.0006451612903225806,
+  "Main_Event": 0.0,
+  "Event_Names": 0.0,
+  "Hazards": 0.0,
+  "Total_Deaths_Min": 0.0,
+  "Total_Deaths_Max": 0.0,
+  "Total_Injuries_Max": 0.0,
+  "Total_Injuries_Min": 0.0,
+  "Total_Buildings_Damaged_Min": 0.0,
+  "Total_Buildings_Damaged_Max": 0.02,
+  "Total_Affected_Min": 0.0,
+  "Total_Affected_Max": 0.0,
+  "Total_Homeless_Min": 0.0,
+  "Total_Homeless_Max": 0.0,
+  "Total_Displaced_Min": 0.0,
+  "Total_Displaced_Max": 0.0,
+  "Total_Damage_Min": 0.0,
+  "Total_Damage_Max": 0.0,
+  "Total_Damage_Unit": 0.0,
+  "Total_Damage_Inflation_Adjusted": 0.0,
+  "Total_Damage_Inflation_Adjusted_Year": 0.0,
+  "Total_Insured_Damage_Min": 0.0,
+  "Total_Insured_Damage_Max": 0.0,
+  "Total_Insured_Damage_Unit": 0.0,
+  "Total_Insured_Damage_Inflation_Adjusted": 0.0,
+  "Total_Insured_Damage_Inflation_Adjusted_Year": 0.0,
+  "Start_Date_Day": 0.0,
+  "Start_Date_Month": 0.0,
+  "Start_Date_Year": 0.0,
+  "End_Date_Day": 0.0,
+  "End_Date_Month": 0.0,
+  "End_Date_Year": 0.0,
+  "Administrative_Areas_Norm": 0.0
+}
diff --git a/Database/evaluation_results/double_annotation/test/l1/l1_all_10_results.csv b/Database/evaluation_results/double_annotation/test/l1/l1_all_10_results.csv
diff --git a/Database/evaluation_results/double_annotation/test/l1/sys_matched_currencies_l1.parquet b/Database/evaluation_results/double_annotation/test/l1/sys_matched_currencies_l1.parquet
diff --git a/...results/double_annotation/test/l2/gold_Instance_Per_Administrative_Areas_Affected.parquet b/...results/double_annotation/test/l2/gold_Instance_Per_Administrative_Areas_Affected.parquet
diff --git a/...ouble_annotation/test/l2/gold_Instance_Per_Administrative_Areas_Buildings_Damaged.parquet b/...ouble_annotation/test/l2/gold_Instance_Per_Administrative_Areas_Buildings_Damaged.parquet
diff --git a/...n_results/double_annotation/test/l2/gold_Instance_Per_Administrative_Areas_Damage.parquet b/...n_results/double_annotation/test/l2/gold_Instance_Per_Administrative_Areas_Damage.parquet
diff --git a/...n_results/double_annotation/test/l2/gold_Instance_Per_Administrative_Areas_Deaths.parquet b/...n_results/double_annotation/test/l2/gold_Instance_Per_Administrative_Areas_Deaths.parquet
diff --git a/...results/double_annotation/test/l2/gold_Instance_Per_Administrative_Areas_Injuries.parquet b/...results/double_annotation/test/l2/gold_Instance_Per_Administrative_Areas_Injuries.parquet
diff --git a/...n/test/l2/l2_all_11_Instance_Per_Administrative_Areas_Damage_avg_per_event_id_results.csv b/...n/test/l2/l2_all_11_Instance_Per_Administrative_Areas_Damage_avg_per_event_id_results.csv
diff --git a/...le_annotation/test/l2/l2_all_11_Instance_Per_Administrative_Areas_Damage_avg_results.json b/...le_annotation/test/l2/l2_all_11_Instance_Per_Administrative_Areas_Damage_avg_results.json
@@ -0,0 +1,16 @@
+{
+  "Coverage": 0.0,
+  "Weighted_Score": 0.5606060606060607,
+  "Start_Date_Day": 0.6363636363636364,
+  "Start_Date_Month": 0.6363636363636364,
+  "Start_Date_Year": 0.5454545454545454,
+  "End_Date_Day": 0.5454545454545454,
+  "End_Date_Month": 0.5454545454545454,
+  "End_Date_Year": 0.5454545454545454,
+  "Administrative_Areas_Norm": 0.5454545454545454,
+  "Num_Min": 0.5454545454545454,
+  "Num_Max": 0.5454545454545454,
+  "Num_Unit": 0.5454545454545454,
+  "Num_Inflation_Adjusted": 0.6363636363636364,
+  "Num_Inflation_Adjusted_Year": 0.45454545454545453
+}
diff --git a/.../double_annotation/test/l2/l2_all_11_Instance_Per_Administrative_Areas_Damage_results.csv b/.../double_annotation/test/l2/l2_all_11_Instance_Per_Administrative_Areas_Damage_results.csv
diff --git a/...2_all_12_Instance_Per_Administrative_Areas_Buildings_Damaged_avg_per_event_id_results.csv b/...2_all_12_Instance_Per_Administrative_Areas_Buildings_Damaged_avg_per_event_id_results.csv
diff --git a/...on/test/l2/l2_all_12_Instance_Per_Administrative_Areas_Buildings_Damaged_avg_results.json b/...on/test/l2/l2_all_12_Instance_Per_Administrative_Areas_Buildings_Damaged_avg_results.json
@@ -0,0 +1,13 @@
+{
+  "Coverage": 0.0,
+  "Weighted_Score": 0.48521027054184146,
+  "Start_Date_Day": 0.25,
+  "Start_Date_Month": 0.25,
+  "Start_Date_Year": 0.8333333333333334,
+  "End_Date_Day": 0.16666666666666666,
+  "End_Date_Month": 0.16666666666666666,
+  "End_Date_Year": 0.16666666666666666,
+  "Administrative_Areas_Norm": 0.8333333333333334,
+  "Num_Min": 0.8501128841049529,
+  "Num_Max": 0.8501128841049529
+}
diff --git a/...otation/test/l2/l2_all_12_Instance_Per_Administrative_Areas_Buildings_Damaged_results.csv b/...otation/test/l2/l2_all_12_Instance_Per_Administrative_Areas_Buildings_Damaged_results.csv
diff --git a/...n/test/l2/l2_all_17_Instance_Per_Administrative_Areas_Deaths_avg_per_event_id_results.csv b/...n/test/l2/l2_all_17_Instance_Per_Administrative_Areas_Deaths_avg_per_event_id_results.csv
diff --git a/...le_annotation/test/l2/l2_all_17_Instance_Per_Administrative_Areas_Deaths_avg_results.json b/...le_annotation/test/l2/l2_all_17_Instance_Per_Administrative_Areas_Deaths_avg_results.json
@@ -0,0 +1,13 @@
+{
+  "Coverage": 0.0,
+  "Weighted_Score": 0.5312445954905423,
+  "Start_Date_Day": 0.4740484429065744,
+  "Start_Date_Month": 0.47058823529411764,
+  "Start_Date_Year": 0.5882352941176471,
+  "End_Date_Day": 0.4980392156862745,
+  "End_Date_Month": 0.5058823529411764,
+  "End_Date_Year": 0.47060287165376236,
+  "Administrative_Areas_Norm": 0.6470588235294118,
+  "Num_Min": 0.5622837370242214,
+  "Num_Max": 0.5644623862616943
+}
diff --git a/.../double_annotation/test/l2/l2_all_17_Instance_Per_Administrative_Areas_Deaths_results.csv b/.../double_annotation/test/l2/l2_all_17_Instance_Per_Administrative_Areas_Deaths_results.csv
diff --git a/.../test/l2/l2_all_2_Instance_Per_Administrative_Areas_Affected_avg_per_event_id_results.csv b/.../test/l2/l2_all_2_Instance_Per_Administrative_Areas_Affected_avg_per_event_id_results.csv
diff --git a/...e_annotation/test/l2/l2_all_2_Instance_Per_Administrative_Areas_Affected_avg_results.json b/...e_annotation/test/l2/l2_all_2_Instance_Per_Administrative_Areas_Affected_avg_results.json
@@ -0,0 +1,13 @@
+{
+  "Coverage": 0.0,
+  "Weighted_Score": 0.501916278455083,
+  "Start_Date_Day": 0.5,
+  "Start_Date_Month": 0.5,
+  "Start_Date_Year": 0.5,
+  "End_Date_Day": 0.5,
+  "End_Date_Month": 0.5,
+  "End_Date_Year": 0.5,
+  "Administrative_Areas_Norm": 0.5,
+  "Num_Min": 0.5087719298245614,
+  "Num_Max": 0.5084745762711864
+}
diff --git a/...double_annotation/test/l2/l2_all_2_Instance_Per_Administrative_Areas_Affected_results.csv b/...double_annotation/test/l2/l2_all_2_Instance_Per_Administrative_Areas_Affected_results.csv
diff --git a/.../test/l2/l2_all_8_Instance_Per_Administrative_Areas_Injuries_avg_per_event_id_results.csv b/.../test/l2/l2_all_8_Instance_Per_Administrative_Areas_Injuries_avg_per_event_id_results.csv
diff --git a/...e_annotation/test/l2/l2_all_8_Instance_Per_Administrative_Areas_Injuries_avg_results.json b/...e_annotation/test/l2/l2_all_8_Instance_Per_Administrative_Areas_Injuries_avg_results.json
@@ -0,0 +1,13 @@
+{
+  "Coverage": 0.0,
+  "Weighted_Score": 0.44619081964156093,
+  "Start_Date_Day": 0.25735294117647056,
+  "Start_Date_Month": 0.25,
+  "Start_Date_Year": 0.75,
+  "End_Date_Day": 0.30833333333333335,
+  "End_Date_Month": 0.325,
+  "End_Date_Year": 0.25003110226424485,
+  "Administrative_Areas_Norm": 0.625,
+  "Num_Min": 0.625,
+  "Num_Max": 0.625
+}
diff --git a/...double_annotation/test/l2/l2_all_8_Instance_Per_Administrative_Areas_Injuries_results.csv b/...double_annotation/test/l2/l2_all_8_Instance_Per_Administrative_Areas_Injuries_results.csv
diff --git a/..._results/double_annotation/test/l2/sys_Instance_Per_Administrative_Areas_Affected.parquet b/..._results/double_annotation/test/l2/sys_Instance_Per_Administrative_Areas_Affected.parquet
diff --git a/...double_annotation/test/l2/sys_Instance_Per_Administrative_Areas_Buildings_Damaged.parquet b/...double_annotation/test/l2/sys_Instance_Per_Administrative_Areas_Buildings_Damaged.parquet
diff --git a/...on_results/double_annotation/test/l2/sys_Instance_Per_Administrative_Areas_Damage.parquet b/...on_results/double_annotation/test/l2/sys_Instance_Per_Administrative_Areas_Damage.parquet
diff --git a/...on_results/double_annotation/test/l2/sys_Instance_Per_Administrative_Areas_Deaths.parquet b/...on_results/double_annotation/test/l2/sys_Instance_Per_Administrative_Areas_Deaths.parquet
diff --git a/..._results/double_annotation/test/l2/sys_Instance_Per_Administrative_Areas_Injuries.parquet b/..._results/double_annotation/test/l2/sys_Instance_Per_Administrative_Areas_Injuries.parquet
diff --git a/...notation/test/l3/gold_Specific_Instance_Per_Administrative_Area_Buildings_Damaged.parquet b/...notation/test/l3/gold_Specific_Instance_Per_Administrative_Area_Buildings_Damaged.parquet
diff --git a/...s/double_annotation/test/l3/gold_Specific_Instance_Per_Administrative_Area_Damage.parquet b/...s/double_annotation/test/l3/gold_Specific_Instance_Per_Administrative_Area_Damage.parquet
diff --git a/...s/double_annotation/test/l3/gold_Specific_Instance_Per_Administrative_Area_Deaths.parquet b/...s/double_annotation/test/l3/gold_Specific_Instance_Per_Administrative_Area_Deaths.parquet
diff --git a/...ouble_annotation/test/l3/gold_Specific_Instance_Per_Administrative_Area_Displaced.parquet b/...ouble_annotation/test/l3/gold_Specific_Instance_Per_Administrative_Area_Displaced.parquet
diff --git a/...double_annotation/test/l3/gold_Specific_Instance_Per_Administrative_Area_Injuries.parquet b/...double_annotation/test/l3/gold_Specific_Instance_Per_Administrative_Area_Injuries.parquet
diff --git a/..._Specific_Instance_Per_Administrative_Area_Buildings_Damaged_avg_per_event_id_results.csv b/..._Specific_Instance_Per_Administrative_Area_Buildings_Damaged_avg_per_event_id_results.csv
diff --git a/...l3/l3_all_19_Specific_Instance_Per_Administrative_Area_Buildings_Damaged_avg_results.json b/...l3/l3_all_19_Specific_Instance_Per_Administrative_Area_Buildings_Damaged_avg_results.json
@@ -0,0 +1,14 @@
+{
+  "Coverage": 0.0,
+  "Weighted_Score": 0.8055956734446409,
+  "Start_Date_Day": 0.9473684210526315,
+  "Start_Date_Month": 0.9473684210526315,
+  "Start_Date_Year": 0.5263157894736842,
+  "End_Date_Day": 0.9473684210526315,
+  "End_Date_Month": 0.9473684210526315,
+  "End_Date_Year": 0.9473684210526315,
+  "Administrative_Area_Norm": 0.6842105263157895,
+  "Locations_Norm": 0.868421052631579,
+  "Num_Min": 0.6405003050875255,
+  "Num_Max": 0.5996669556746719
+}
diff --git a/...test/l3/l3_all_19_Specific_Instance_Per_Administrative_Area_Buildings_Damaged_results.csv b/...test/l3/l3_all_19_Specific_Instance_Per_Administrative_Area_Buildings_Damaged_results.csv
diff --git a/...3/l3_all_22_Specific_Instance_Per_Administrative_Area_Deaths_avg_per_event_id_results.csv b/...3/l3_all_22_Specific_Instance_Per_Administrative_Area_Deaths_avg_per_event_id_results.csv
diff --git a/...ation/test/l3/l3_all_22_Specific_Instance_Per_Administrative_Area_Deaths_avg_results.json b/...ation/test/l3/l3_all_22_Specific_Instance_Per_Administrative_Area_Deaths_avg_results.json
@@ -0,0 +1,14 @@
+{
+  "Coverage": 0.0,
+  "Weighted_Score": 0.5629708532044584,
+  "Start_Date_Day": 0.6778826728826729,
+  "Start_Date_Month": 0.5909090909090909,
+  "Start_Date_Year": 0.18181818181818182,
+  "End_Date_Day": 0.7727272727272727,
+  "End_Date_Month": 0.7727272727272727,
+  "End_Date_Year": 0.7727272727272727,
+  "Administrative_Area_Norm": 0.45454545454545453,
+  "Locations_Norm": 0.7272727272727273,
+  "Num_Min": 0.3392316602688719,
+  "Num_Max": 0.3398669261657657
+}
diff --git a/...annotation/test/l3/l3_all_22_Specific_Instance_Per_Administrative_Area_Deaths_results.csv b/...annotation/test/l3/l3_all_22_Specific_Instance_Per_Administrative_Area_Deaths_results.csv
diff --git a/...l3/l3_all_5_Specific_Instance_Per_Administrative_Area_Damage_avg_per_event_id_results.csv b/...l3/l3_all_5_Specific_Instance_Per_Administrative_Area_Damage_avg_per_event_id_results.csv
diff --git a/...tation/test/l3/l3_all_5_Specific_Instance_Per_Administrative_Area_Damage_avg_results.json b/...tation/test/l3/l3_all_5_Specific_Instance_Per_Administrative_Area_Damage_avg_results.json
@@ -0,0 +1,17 @@
+{
+  "Coverage": 0.0,
+  "Weighted_Score": 0.5461538461538461,
+  "Start_Date_Day": 1.0,
+  "Start_Date_Month": 1.0,
+  "Start_Date_Year": 0.4,
+  "End_Date_Day": 1.0,
+  "End_Date_Month": 1.0,
+  "End_Date_Year": 1.0,
+  "Administrative_Area_Norm": 0.4,
+  "Locations_Norm": 0.7,
+  "Num_Min": 0.0,
+  "Num_Max": 0.0,
+  "Num_Unit": 0.0,
+  "Num_Inflation_Adjusted": 0.4,
+  "Num_Inflation_Adjusted_Year": 0.2
+}
diff --git a/..._annotation/test/l3/l3_all_5_Specific_Instance_Per_Administrative_Area_Damage_results.csv b/..._annotation/test/l3/l3_all_5_Specific_Instance_Per_Administrative_Area_Damage_results.csv
diff --git a/...l3_all_8_Specific_Instance_Per_Administrative_Area_Displaced_avg_per_event_id_results.csv b/...l3_all_8_Specific_Instance_Per_Administrative_Area_Displaced_avg_per_event_id_results.csv
diff --git a/...ion/test/l3/l3_all_8_Specific_Instance_Per_Administrative_Area_Displaced_avg_results.json b/...ion/test/l3/l3_all_8_Specific_Instance_Per_Administrative_Area_Displaced_avg_results.json
@@ -0,0 +1,14 @@
+{
+  "Coverage": 0.0,
+  "Weighted_Score": 0.875,
+  "Start_Date_Day": 1.0,
+  "Start_Date_Month": 1.0,
+  "Start_Date_Year": 0.75,
+  "End_Date_Day": 1.0,
+  "End_Date_Month": 1.0,
+  "End_Date_Year": 1.0,
+  "Administrative_Area_Norm": 0.75,
+  "Locations_Norm": 0.75,
+  "Num_Min": 0.75,
+  "Num_Max": 0.75
+}
diff --git a/...notation/test/l3/l3_all_8_Specific_Instance_Per_Administrative_Area_Displaced_results.csv b/...notation/test/l3/l3_all_8_Specific_Instance_Per_Administrative_Area_Displaced_results.csv
diff --git a/.../l3_all_8_Specific_Instance_Per_Administrative_Area_Injuries_avg_per_event_id_results.csv b/.../l3_all_8_Specific_Instance_Per_Administrative_Area_Injuries_avg_per_event_id_results.csv
diff --git a/...tion/test/l3/l3_all_8_Specific_Instance_Per_Administrative_Area_Injuries_avg_results.json b/...tion/test/l3/l3_all_8_Specific_Instance_Per_Administrative_Area_Injuries_avg_results.json
@@ -0,0 +1,14 @@
+{
+  "Coverage": 0.0,
+  "Weighted_Score": 0.6604924242424243,
+  "Start_Date_Day": 0.625,
+  "Start_Date_Month": 0.625,
+  "Start_Date_Year": 0.625,
+  "End_Date_Day": 0.625,
+  "End_Date_Month": 0.625,
+  "End_Date_Year": 0.625,
+  "Administrative_Area_Norm": 0.625,
+  "Locations_Norm": 0.875,
+  "Num_Min": 0.6840909090909091,
+  "Num_Max": 0.6708333333333334
+}
diff --git a/...nnotation/test/l3/l3_all_8_Specific_Instance_Per_Administrative_Area_Injuries_results.csv b/...nnotation/test/l3/l3_all_8_Specific_Instance_Per_Administrative_Area_Injuries_results.csv
diff --git a/...nnotation/test/l3/sys_Specific_Instance_Per_Administrative_Area_Buildings_Damaged.parquet b/...nnotation/test/l3/sys_Specific_Instance_Per_Administrative_Area_Buildings_Damaged.parquet
diff --git a/...ts/double_annotation/test/l3/sys_Specific_Instance_Per_Administrative_Area_Damage.parquet b/...ts/double_annotation/test/l3/sys_Specific_Instance_Per_Administrative_Area_Damage.parquet
diff --git a/...ts/double_annotation/test/l3/sys_Specific_Instance_Per_Administrative_Area_Deaths.parquet b/...ts/double_annotation/test/l3/sys_Specific_Instance_Per_Administrative_Area_Deaths.parquet
diff --git a/...double_annotation/test/l3/sys_Specific_Instance_Per_Administrative_Area_Displaced.parquet b/...double_annotation/test/l3/sys_Specific_Instance_Per_Administrative_Area_Displaced.parquet
diff --git a/.../double_annotation/test/l3/sys_Specific_Instance_Per_Administrative_Area_Injuries.parquet b/.../double_annotation/test/l3/sys_Specific_Instance_Per_Administrative_Area_Injuries.parquet
diff --git a/Database/gold/ImpactDB_DataTable_Validation.xlsx b/Database/gold/ImpactDB_DataTable_Validation.xlsx
diff --git a/Database/gold/double_annotation/test/l1/Total_Summary.parquet b/Database/gold/double_annotation/test/l1/Total_Summary.parquet
diff --git a/Database/gold/double_annotation/test/l2/Instance_Per_Administrative_Areas_Affected.parquet b/Database/gold/double_annotation/test/l2/Instance_Per_Administrative_Areas_Affected.parquet
diff --git a/...old/double_annotation/test/l2/Instance_Per_Administrative_Areas_Buildings_Damaged.parquet b/...old/double_annotation/test/l2/Instance_Per_Administrative_Areas_Buildings_Damaged.parquet
diff --git a/Database/gold/double_annotation/test/l2/Instance_Per_Administrative_Areas_Damage.parquet b/Database/gold/double_annotation/test/l2/Instance_Per_Administrative_Areas_Damage.parquet
diff --git a/Database/gold/double_annotation/test/l2/Instance_Per_Administrative_Areas_Deaths.parquet b/Database/gold/double_annotation/test/l2/Instance_Per_Administrative_Areas_Deaths.parquet
diff --git a/Database/gold/double_annotation/test/l2/Instance_Per_Administrative_Areas_Injuries.parquet b/Database/gold/double_annotation/test/l2/Instance_Per_Administrative_Areas_Injuries.parquet
diff --git a/...gold/double_annotation/test/l3/Specific_Instance_Per_Administrative_Area_Affected.parquet b/...gold/double_annotation/test/l3/Specific_Instance_Per_Administrative_Area_Affected.parquet
diff --git a/...le_annotation/test/l3/Specific_Instance_Per_Administrative_Area_Buildings_Damaged.parquet b/...le_annotation/test/l3/Specific_Instance_Per_Administrative_Area_Buildings_Damaged.parquet
Original file line number	Diff line number	Diff line change
Expand Up		@@ -12,3 +12,5 @@ setting 5 & 1.0 for location, 0 for others & 0.6 & 1.0 \

		Based on the evaluation result of V3 cross these experiments, we select setting 2 for the further processing.
		For the V4 and V5, we only use setting 2 for evaluation.

		For double annotation, we have two annotators: one for the gold data and one for the output data. The annotator for the gold data worked prior to the implementation of the L1-3 guideline, while the annotator for the output data followed the new guideline. The L2 and L3 information in the gold data is extracted from the Location field. Information at the country level is assigned to L2, while sub-national level information is assigned to L3.