[TTS] Infrastructure for parallelization of evaluation (scoring)

examples/tts/magpietts_inference.py

@@ -59,7 +59,6 @@
 # Import the modular components
 from nemo.collections.tts.modules.magpietts_inference.evaluation import (
     DEFAULT_VIOLIN_METRICS,
-    STANDARD_METRIC_KEYS,
     EvaluationConfig,
     compute_mean_with_confidence_interval,
     evaluate_generated_audio_dir,
@@ -292,6 +291,7 @@ def run_inference_and_evaluation(
                 with_utmosv2=eval_config.with_utmosv2,
                 with_fcd=eval_config.with_fcd,
                 codec_model_path=eval_config.codec_model_path,
+                device=eval_config.device,
             )
 
             metrics, filewise_metrics = evaluate_generated_audio_dir(
@@ -308,8 +308,9 @@ def run_inference_and_evaluation(
             with open(os.path.join(eval_dir, f"{dataset}_metrics_{repeat_idx}.json"), "w") as f:
                 json.dump(metrics, f, indent=4)
 
+            sorted_filewise = sorted(filewise_metrics, key=lambda x: x.get('cer', 0), reverse=True)
             with open(os.path.join(eval_dir, f"{dataset}_filewise_metrics_{repeat_idx}.json"), "w") as f:
-                json.dump(filewise_metrics, f, indent=4)
+                json.dump(sorted_filewise, f, indent=4)
 
             # Append to per-run CSV
             append_metrics_to_csv(per_run_csv, full_checkpoint_name, dataset, metrics)
@@ -331,7 +332,6 @@ def run_inference_and_evaluation(
         # Compute mean with confidence interval across repeats
         metrics_mean_ci = compute_mean_with_confidence_interval(
             metrics_all_repeats,
-            STANDARD_METRIC_KEYS,
             confidence=confidence_level,
         )
 

nemo/collections/tts/modules/magpietts_inference/__init__.py

@@ -45,7 +45,6 @@
 
 from nemo.collections.tts.modules.magpietts_inference.evaluation import (
     DEFAULT_VIOLIN_METRICS,
-    STANDARD_METRIC_KEYS,
     EvaluationConfig,
     compute_mean_with_confidence_interval,
     evaluate_generated_audio_dir,
@@ -74,7 +73,6 @@
     "EvaluationConfig",
     "evaluate_generated_audio_dir",
     "compute_mean_with_confidence_interval",
-    "STANDARD_METRIC_KEYS",
     "DEFAULT_VIOLIN_METRICS",
     # Visualization
     "create_violin_plot",