fix(analysis): keep matrix rounds distinct in bug-event charts

analysis/analyze.py

@@ -25,9 +25,17 @@ class LogFile:
     path: Path
     instance_id: str
     fuzzer_label: str
+    run_id: Optional[str] = None
 
 LOG_FILE_RE = re.compile(r".+\.log$")
 INSTANCE_PREFIX_RE = re.compile(r"^(i-[0-9a-f]+)-(.*)$")
+# Matrix/aggregated benchmark runs are flattened by the workflows collector
+# (modules/tests.nu `scfuzzbench-collect-all-logs`) into a single per-target
+# directory. Each round's log files are renamed to "<run_id>-<original_name>",
+# where <run_id> is the safe-labeled path of that round's logs.zip and therefore
+# always ends in "logs.zip". Recovering this per-round prefix keeps multiple
+# rounds from collapsing into a single shared "unknown" run.
+MATRIX_RUN_PREFIX_RE = re.compile(r"^(?P<run_id>.*logs\.zip)-(?P<name>.+)$")
 IGNORED_LOG_FILENAMES = {"runner_commands.log"}
 ABS_TS_RE = re.compile(r"^\[(\d{4}-\d{2}-\d{2} [0-9:.]+)\]")
 MEDUSA_ELAPSED_RE = re.compile(r"elapsed:\s*([0-9hms]+)")
@@ -925,7 +933,9 @@ def discover_log_files(logs_dir: Path) -> Tuple[LogFile, ...]:
             continue
         instance_label = rel.parts[0]
         instance_id, fuzzer_label = split_instance_label(instance_label)
-        files.append(LogFile(path, instance_id, fuzzer_label))
+        matrix_match = MATRIX_RUN_PREFIX_RE.match(path.name)
+        file_run_id = matrix_match.group("run_id") if matrix_match else None
+        files.append(LogFile(path, instance_id, fuzzer_label, file_run_id))
     return tuple(files)
 
 
@@ -935,11 +945,12 @@ def parse_logs(
     log_files: Sequence[LogFile],
 ) -> List[Event]:
     events: List[Event] = []
-    run_id_value = run_id or infer_run_id(logs_dir) or "unknown"
+    fallback_run_id = run_id or infer_run_id(logs_dir) or "unknown"
     for log_file in log_files:
         path = log_file.path
         instance_id = log_file.instance_id
         fuzzer_label = log_file.fuzzer_label
+        run_id_value = run_id or log_file.run_id or fallback_run_id
         fuzzer = normalize_fuzzer(fuzzer_label)
         if fuzzer == "foundry":
             events.extend(parse_foundry_log(path, run_id_value, instance_id, fuzzer_label))
@@ -980,8 +991,9 @@ def parse_throughput_logs(
     log_files: Sequence[LogFile],
 ) -> List[ThroughputSample]:
     samples: List[ThroughputSample] = []
-    run_id_value = run_id or infer_run_id(logs_dir) or "unknown"
+    fallback_run_id = run_id or infer_run_id(logs_dir) or "unknown"
     for log_file in log_files:
+        run_id_value = run_id or log_file.run_id or fallback_run_id
         samples.extend(
             parse_throughput_log(
                 log_file.path,
@@ -1120,8 +1132,9 @@ def parse_progress_metrics_logs(
     log_files: Sequence[LogFile],
 ) -> List[ProgressMetricsSample]:
     samples: List[ProgressMetricsSample] = []
-    run_id_value = run_id or infer_run_id(logs_dir) or "unknown"
+    fallback_run_id = run_id or infer_run_id(logs_dir) or "unknown"
     for log_file in log_files:
+        run_id_value = run_id or log_file.run_id or fallback_run_id
         samples.extend(
             parse_progress_metrics_log(
                 log_file.path,

analysis/events_to_cumulative.py

@@ -14,6 +14,13 @@
 }
 
 INSTANCE_PREFIX_RE = re.compile(r"^(i-[0-9a-f]+)-(.*)$")
+# Matrix/aggregated runs are flattened by the workflows collector into one
+# per-target directory, with each round's log files renamed to
+# "<run_id>-<original_name>" where <run_id> ends in "logs.zip". Recover that
+# per-round prefix so zero-event seeding produces one entry per round instead of
+# collapsing every round into a single "unknown" run. Mirrors analyze.py.
+MATRIX_RUN_PREFIX_RE = re.compile(r"^(?P<run_id>.*logs\.zip)-(?P<name>.+)$")
+IGNORED_LOG_FILENAMES = {"runner_commands.log"}
 
 
 def die(msg: str) -> None:
@@ -68,15 +75,29 @@ def inventory_runs_from_logs(
     exclude_fuzzers: Optional[set[str]] = None,
     raw_labels: bool = False,
 ) -> List[Tuple[str, str]]:
-    run_id_value = run_id or infer_run_id(logs_dir) or "unknown"
+    fallback_run_id = run_id or infer_run_id(logs_dir) or "unknown"
     runs: List[Tuple[str, str]] = []
-    for instance_dir in sorted([p for p in logs_dir.iterdir() if p.is_dir()]):
-        instance_id, fuzzer_label = split_instance_label(instance_dir.name)
+    seen: set[Tuple[str, str]] = set()
+    for path in sorted(logs_dir.rglob("*")):
+        if not path.is_file() or not path.name.endswith(".log"):
+            continue
+        if path.name in IGNORED_LOG_FILENAMES:
+            continue
+        rel = path.relative_to(logs_dir)
+        if len(rel.parts) < 2:
+            continue
+        instance_id, fuzzer_label = split_instance_label(rel.parts[0])
         fuzzer = fuzzer_label if raw_labels else normalize_fuzzer(fuzzer_label)
         if exclude_fuzzers:
             if str(fuzzer).lower() in exclude_fuzzers or fuzzer_label.lower() in exclude_fuzzers:
                 continue
-        runs.append((fuzzer, f"{run_id_value}:{instance_id}"))
+        matrix_match = MATRIX_RUN_PREFIX_RE.match(path.name)
+        file_run_id = run_id or (matrix_match.group("run_id") if matrix_match else None) or fallback_run_id
+        entry = (fuzzer, f"{file_run_id}:{instance_id}")
+        if entry in seen:
+            continue
+        seen.add(entry)
+        runs.append(entry)
     return runs
 
 

analysis/tests/test_matrix_run_identity.py

@@ -0,0 +1,123 @@
+import tempfile
+import unittest
+from pathlib import Path
+
+from analysis import analyze
+from analysis import events_to_cumulative
+
+
+# Matrix/aggregated benchmark runs are flattened by the workflows collector
+# (modules/tests.nu `scfuzzbench-collect-all-logs`) into a single per-target
+# directory. Each round's log files are renamed to "<run_id>-<original_name>",
+# where <run_id> is the safe-labeled path of that round's logs.zip and therefore
+# always ends in "logs.zip". These tests pin the contract so multiple rounds are
+# never collapsed into a single shared "unknown" run.
+def matrix_filename(round_seg: str, target: str, name: str) -> str:
+    return f"-{target}-foundry-master-{round_seg}-logs.zip-{name}"
+
+
+FAILURE_LINE = (
+    '{{"timestamp":{ts},"event":"failure",'
+    '"target":"CryticToFoundry:invariant_a","type":"invariant"}}'
+)
+
+
+class MatrixRunIdentityTests(unittest.TestCase):
+    def _write_matrix_logs(self, logs_dir: Path) -> None:
+        # Two rounds for a target that finds a bug.
+        aave = logs_dir / "foundry-master__target-aave-v4"
+        aave.mkdir(parents=True)
+        for round_seg in ("round-1", "round-2"):
+            (aave / matrix_filename(round_seg, "aave-v4", "foundry.log")).write_text(
+                FAILURE_LINE.format(ts=100) + "\n" + FAILURE_LINE.format(ts=200) + "\n",
+                encoding="utf-8",
+            )
+
+        # Two rounds for a target with no bug events (the zero-bug case that
+        # previously collapsed to a single seeded "unknown" run).
+        liquity = logs_dir / "foundry-master__target-liquity-v2-governance"
+        liquity.mkdir(parents=True)
+        for round_seg in ("round-1", "round-2"):
+            (liquity / matrix_filename(round_seg, "liquity-v2-governance", "foundry.log")).write_text(
+                '{"timestamp":100,"event":"pulse","metrics":{"cumulative_edges_seen":1}}\n',
+                encoding="utf-8",
+            )
+
+    def test_discover_recovers_per_round_run_id(self):
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            logs_dir = Path(tmp_dir)
+            self._write_matrix_logs(logs_dir)
+
+            log_files = analyze.discover_log_files(logs_dir)
+            run_ids = {lf.run_id for lf in log_files}
+            # Each of the 4 round files must carry a distinct, non-null run id.
+            self.assertEqual(len(run_ids), 4)
+            self.assertNotIn(None, run_ids)
+            for run_id in run_ids:
+                self.assertTrue(run_id.endswith("logs.zip"))
+
+    def test_parse_logs_keeps_rounds_distinct(self):
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            logs_dir = Path(tmp_dir)
+            self._write_matrix_logs(logs_dir)
+
+            log_files = analyze.discover_log_files(logs_dir)
+            events = analyze.parse_logs(logs_dir, None, log_files)
+
+            aave_runs = {
+                e.run_id for e in events if "aave-v4" in e.fuzzer_label
+            }
+            # The bug-finding target must show two distinct rounds, not one
+            # collapsed "unknown" run.
+            self.assertEqual(len(aave_runs), 2)
+            self.assertNotIn("unknown", aave_runs)
+
+    def test_inventory_seeds_one_entry_per_round(self):
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            logs_dir = Path(tmp_dir)
+            self._write_matrix_logs(logs_dir)
+
+            runs = events_to_cumulative.inventory_runs_from_logs(
+                logs_dir=logs_dir, run_id=None, raw_labels=True
+            )
+            liquity_keys = {
+                run_key
+                for fuzzer, run_key in runs
+                if "liquity" in fuzzer
+            }
+            # Zero-bug target must still seed two distinct rounds.
+            self.assertEqual(len(liquity_keys), 2)
+            self.assertTrue(all(not k.startswith("unknown:") for k in liquity_keys))
+
+    def test_cumulative_run_keys_align_between_events_and_inventory(self):
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            logs_dir = Path(tmp_dir)
+            self._write_matrix_logs(logs_dir)
+
+            log_files = analyze.discover_log_files(logs_dir)
+            events = analyze.parse_logs(logs_dir, None, log_files)
+            event_dicts = [
+                {
+                    "run_id": e.run_id,
+                    "instance_id": e.instance_id,
+                    "fuzzer": e.fuzzer_label,
+                    "fuzzer_label": e.fuzzer_label,
+                    "elapsed_seconds": e.elapsed_seconds,
+                }
+                for e in events
+            ]
+
+            rows = events_to_cumulative.build_cumulative_rows(
+                event_dicts,
+                include_zero=True,
+                logs_dir=logs_dir,
+                run_id=None,
+                raw_labels=True,
+            )
+            # 4 rounds total across both targets => 4 distinct cumulative series.
+            run_keys = {(fuzzer, run_key) for fuzzer, run_key, _, _ in rows}
+            self.assertEqual(len(run_keys), 4)
+
+
+if __name__ == "__main__":
+    unittest.main()