Fix Vocal Sound reference for throat clearing (#3325)

Co-authored-by: Haoqin Tu <[email protected]>
stanford-crfm · Feb 7, 2025 · 6189234 · 6189234
1 parent e34ec7b
commit 6189234
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 1 deletion.
diff --git a/...mark/presentation/run_entries_speech.conf → ...hmark/presentation/run_entries_audio.conf b/...mark/presentation/run_entries_speech.conf → ...hmark/presentation/run_entries_audio.conf
diff --git a/src/helm/benchmark/scenarios/audio_language/vocal_sound_scenario.py b/src/helm/benchmark/scenarios/audio_language/vocal_sound_scenario.py
@@ -56,10 +56,14 @@ def get_instances(self, output_path: str) -> List[Instance]:
             if not file_name.endswith(".wav") or is_invalid_audio_file(local_audio_path):
                 continue
 
-            answer: str = file_name.split("_")[-1].split(".")[0]
             input = Input(
                 multimedia_content=MultimediaObject([MediaObject(content_type="audio/wav", location=local_audio_path)])
             )
+
+            answer: str = file_name.split("_")[-1].split(".")[0]
+            if answer == "throatclearing":
+                answer = "throat clearing"
+
             references = [Reference(Output(text=str(answer)), tags=[CORRECT_TAG])]
             instances.append(Instance(input=input, references=references, split=TEST_SPLIT))
         return instances