Fix scoring lifecycle and evaluator validation gaps

.github/workflows/validate.yml

@@ -47,11 +47,33 @@ jobs:
         run: |
           .venv/bin/python -c 'import sys; assert sys.prefix != sys.base_prefix'
           .venv/bin/python -c 'import numpy, scipy, sklearn; print(numpy.__version__, scipy.__version__, sklearn.__version__)'
+      - name: Verify TOPS-fMRI metric baseline contract
+        run: .venv/bin/python -m unittest tasks/tops-fmri/checks/test_metrics.py
       - name: Verify bciciv-2a preprocessing contract
         run: .venv/bin/python -m unittest tasks/bciciv-2a/checks/test_preprocessing_contract.py
       - name: Verify sleep-edf preprocessing contract
         run: .venv/bin/python -m unittest tasks/sleep-edf/checks/test_preprocessing_contract.py
 
+  eeg-real-path-smoke:
+    name: EEG real-path smoke (Ubuntu, Python 3.12)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Install pinned EEG evaluator dependencies
+        run: |
+          # CPU-only CI smoke: avoid downloading the multi-GB CUDA runtime used by official GPU scoring.
+          python3 -m pip install --disable-pip-version-check \
+            --index-url https://download.pytorch.org/whl/cpu "torch==2.7.0"
+          python3 -m pip install --disable-pip-version-check \
+            -r tasks/bciciv-2a/env/requirements.txt
+      - name: Exercise BCI preprocessing, model import, training, inference, and metrics
+        run: python3 -m unittest tasks/bciciv-2a/checks/test_real_path_smoke.py
+      - name: Exercise Sleep preprocessing, model import, training, inference, and metrics
+        run: python3 -m unittest tasks/sleep-edf/checks/test_real_path_smoke.py
+
   validate:
     strategy:
       fail-fast: false
@@ -82,6 +104,14 @@ jobs:
         env:
           BPB_DOCKER_TEST: ${{ runner.os == 'Linux' && '1' || '0' }}
           BPB_TEST_INFERENCE_IMAGE: ${{ runner.os == 'Linux' && 'bpb-inference-test:ci' || '' }}
+      - name: EEG evaluator Docker mount/isolation smoke
+        if: runner.os == 'Linux'
+        run: |
+          python3 -m unittest tasks/bciciv-2a/checks/test_docker_isolation.py
+          python3 -m unittest tasks/sleep-edf/checks/test_docker_isolation.py
+        env:
+          BPB_DOCKER_TEST: "1"
+          BPB_TEST_INFERENCE_IMAGE: bpb-inference-test:ci
       - name: Documentation command smoke test
         run: npm run test:docs
       - name: canary 检查
@@ -94,13 +124,15 @@ jobs:
   required:
     name: required
     if: always()
-    needs: [python-environment, validate]
+    needs: [python-environment, eeg-real-path-smoke, validate]
     runs-on: ubuntu-latest
     steps:
       - name: Require successful validation jobs
         env:
           PYTHON_RESULT: ${{ needs.python-environment.result }}
+          EEG_RESULT: ${{ needs.eeg-real-path-smoke.result }}
           VALIDATE_RESULT: ${{ needs.validate.result }}
         run: |
           test "$PYTHON_RESULT" = success
+          test "$EEG_RESULT" = success
           test "$VALIDATE_RESULT" = success

README.md

@@ -194,6 +194,9 @@ scoring independent of the agent language, model provider, and orchestration fra
 - Deterministic metrics, held-out evaluation, and expert rubrics are used where appropriate.
 - A valid score of <code>0.00</code>, an unscored run, and a metric that does not apply are
   represented as different states.
+- Artifact validity, Agent lifecycle, and grader outcome are recorded independently. A
+  timed-out run with a valid bundle is still graded, but is excluded from the official
+  leaderboard by default; use <code>--include-ineligible</code> only for diagnostics.
 - Public inputs are pinned by content hash; evaluator-only inputs remain outside the agent
   workspace.
 - Official results are produced from preserved artifacts and run metadata rather than

src/cli-score.test.ts

@@ -0,0 +1,53 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { spawnSync } from "node:child_process";
+
+const CLI = join(process.cwd(), "dist", "cli.js");
+
+function scoreEnv(): NodeJS.ProcessEnv {
+  const env = { ...process.env };
+  delete env.BPB_SUBMISSION_ISOLATION;
+  delete env.BPB_OFFICIAL_SCORING;
+  delete env.BPB_INFERENCE_IMAGE;
+  return env;
+}
+
+function writeBundle(dir: string, includeArtifact: boolean): void {
+  mkdirSync(join(dir, "artifacts"), { recursive: true });
+  writeFileSync(join(dir, "meta.json"), JSON.stringify({ taskId: "example-exec-task", agent: "cli-regression" }));
+  writeFileSync(join(dir, "signals.json"), JSON.stringify({ completed: false, reason: "timeout" }));
+  if (includeArtifact) writeFileSync(join(dir, "artifacts", "results.csv"), "id,score\n1,10\n2,20\n3,30\n");
+}
+
+test("score CLI grades valid artifacts from an incomplete run and marks them ineligible", () => {
+  const dir = mkdtempSync(join(tmpdir(), "bpb-cli-score-incomplete-"));
+  try {
+    writeBundle(dir, true);
+    const result = spawnSync(process.execPath, [CLI, "score", dir], { cwd: process.cwd(), env: scoreEnv(), encoding: "utf8" });
+    assert.equal(result.status, 0, `${result.stderr}\n${result.stdout}`);
+    const scores = JSON.parse(readFileSync(join(dir, "scores.json"), "utf8"));
+    assert.equal(scores.state, "scored");
+    assert.equal(scores.artifactState, "valid");
+    assert.equal(scores.runState, "timeout");
+    assert.equal(scores.graderState, "scored");
+    assert.equal(scores.leaderboardEligible, false);
+    assert.equal(scores.results[0].value.rows_ok, 1);
+  } finally { rmSync(dir, { recursive: true, force: true }); }
+});
+
+test("score CLI still rejects missing artifacts and does not run the grader", () => {
+  const dir = mkdtempSync(join(tmpdir(), "bpb-cli-score-invalid-"));
+  try {
+    writeBundle(dir, false);
+    const result = spawnSync(process.execPath, [CLI, "score", dir], { cwd: process.cwd(), env: scoreEnv(), encoding: "utf8" });
+    assert.equal(result.status, 1, `${result.stderr}\n${result.stdout}`);
+    const scores = JSON.parse(readFileSync(join(dir, "scores.json"), "utf8"));
+    assert.equal(scores.state, "submission_invalid");
+    assert.equal(scores.artifactState, "invalid");
+    assert.equal(scores.graderState, "not_run");
+    assert.deepEqual(scores.results, []);
+  } finally { rmSync(dir, { recursive: true, force: true }); }
+});

src/cli.ts

@@ -30,7 +30,7 @@ import { loadRunScores, buildLeaderboard } from "./leaderboard.js";
 import { renderLeaderboard, type LeaderboardFormat } from "./leaderboard-format.js";
 import { loadCategories } from "./categories.js";
 import { resolveManifest, parseDatasetSelection, selectDatasets, type DatasetSelection } from "./data/index.js";
-import { runScorers, type RunBundle, type RunScores } from "./score.js";
+import { classifyRunScores, runScorers, type RunBundle, type RunScores } from "./score.js";
 import { validateTask } from "./validate.js";
 import { buildRelease, addRelease, loadRegistry, saveRegistry, verifyRegistry, checkFreezeVisibility } from "./registry.js";
 import { loadSubmissionMeta, verifySubmission, type SubmissionMeta } from "./submission.js";
@@ -316,7 +316,9 @@ async function main() {
       console.log(`  artifacts: ${built.artifacts.length} → ${join(runDir, "artifacts")}`);
       for (const issue of built.issues) console.log(`  ${issue.level}: ${issue.msg}`);
       if (errors.length) throw new Error(`submission bundle failed verification (${errors.length} errors): ${runDir}`);
-      console.log(`${G}✓ completed and verified${X} → ${runDir}`);
+      const lifecycle = execution.result!.reason;
+      const lifecycleColor = lifecycle === "completed" ? G : Y;
+      console.log(`${lifecycleColor}✓ bundle verified (run=${lifecycle})${X} → ${runDir}`);
     }
     return;
   }
@@ -333,7 +335,9 @@ async function main() {
     // 默认只渲染 public 任务的行(--visibility heldout|all 切换);按任务 visibility 过滤 runs。
     const visIds = visibleTaskIds();
     const runs = loadRunScores(runsDir).filter((r) => visIds.has(r.taskId));
-    const tables = buildLeaderboard(runs, (id) => catById.get(id), reg);
+    const tables = buildLeaderboard(runs, (id) => catById.get(id), reg, {
+      includeIneligible: argv.includes("--include-ineligible"),
+    });
     if (!tables.length) { console.log("（无 scores.json 或无 category 记录）"); return; }
     const format = arg("--format", "table") as LeaderboardFormat;
     if (!["table", "json", "markdown", "csv"].includes(format)) {
@@ -467,27 +471,30 @@ async function main() {
     const scoredAt = new Date().toISOString();
     const contractIssues = verifySubmission(t, runDir);
     const contractErrors = contractIssues.filter((issue) => issue.level === "error");
-    if (signalsRaw?.completed === false) contractErrors.push({ level: "error", msg: "agent run is not completed" });
     if (contractErrors.length) {
-      const invalid: RunScores = {
+      const invalid = classifyRunScores({
         taskId, runId, version, scoredAt, state: "submission_invalid",
         explanation: contractErrors.map((issue) => issue.msg).join("; "), results: [],
-      };
+      }, "invalid", signalsRaw);
       writeFileSync(join(runDir, "scores.json"), JSON.stringify(invalid, null, 2));
       console.error(`${Y}submission_invalid${X}: ${invalid.explanation}`);
       process.exitCode = 1;
       return;
     }
-    console.log(`${G}ready_to_score${X}: bundle verified; scoring starts after Agent completion`);
+    console.log(`${G}ready_to_score${X}: bundle verified; starting artifact scoring`);
     const bundle: RunBundle = { runDir, runId, version, events, signals: (signalsRaw ?? meta) as Record<string, unknown> };
     let scores: RunScores;
     try {
       scores = await runScorers(t, bundle, scoredAt);
     } catch (error) {
       scores = { taskId, runId, version, scoredAt, state: "scoring_failed", explanation: (error as Error).message, results: [] };
     }
+    scores = classifyRunScores(scores, "valid", signalsRaw);
     writeFileSync(join(runDir, "scores.json"), JSON.stringify(scores, null, 2));
-    console.log(`${B}— score ${taskId}${X}  state=${scores.state} → ${join(runDir, "scores.json")}`);
+    console.log(`${B}— score ${taskId}${X}  grader=${scores.graderState} run=${scores.runState} eligible=${scores.leaderboardEligible} → ${join(runDir, "scores.json")}`);
+    if (!scores.leaderboardEligible && scores.leaderboardExclusionReason) {
+      console.warn(`${Y}leaderboard_ineligible${X}: ${scores.leaderboardExclusionReason}`);
+    }
     if (scores.explanation) console.log(`  ${scores.explanation}`);
     for (const r of scores.results) {
       const v = r.unscored ? `${Y}${r.state ?? "scoring_failed"}${X}` : (typeof r.value === "number" ? String(r.value) : JSON.stringify(r.value));
@@ -559,7 +566,7 @@ async function main() {
     return;
   }
 
-  console.log("用法: bp-bench list | doctor [id] [--private] [--python <path>] [--base-url <url>] [--isolation process|docker] | prepare <id> [--workspace <dir>] [--fetch] | run <id|all> --adapter brainpilot|command|manual [--isolation process|docker] [--agent <id>] [--resume <runDir>] | fetch <id|all> [--public|--private|--all] | score <bundle> [--isolation process|docker] [--inference-image <image>] [--official] [--judge-model <id>] | stats <bundle|runsDir> [--format table|json|markdown|csv] | validate <id|all> [--allow-heldout] | leaderboard <runsDir> [--format table|json|markdown|csv] | freeze <name> [--ref <git-ref>] | registry verify | submit verify <bundle>");
+  console.log("用法: bp-bench list | doctor [id] [--private] [--python <path>] [--base-url <url>] [--isolation process|docker] | prepare <id> [--workspace <dir>] [--fetch] | run <id|all> --adapter brainpilot|command|manual [--isolation process|docker] [--agent <id>] [--resume <runDir>] | fetch <id|all> [--public|--private|--all] | score <bundle> [--isolation process|docker] [--inference-image <image>] [--official] [--judge-model <id>] | stats <bundle|runsDir> [--format table|json|markdown|csv] | validate <id|all> [--allow-heldout] | leaderboard <runsDir> [--format table|json|markdown|csv] [--include-ineligible] | freeze <name> [--ref <git-ref>] | registry verify | submit verify <bundle>");
   console.log("       通用: --tasks <dir1,dir2>(多根) | --visibility public|heldout|all(list/leaderboard/freeze;缺省 public)");
 }
 

src/leaderboard.test.ts

@@ -129,3 +129,39 @@ test("buildLeaderboard: taskId/version 含空格不串味(单射 key + 透传 la
   assert.ok(sw.rows.every((r) => r.taskId === "t1"));
 });
 
+test("buildLeaderboard: 默认排除 lifecycle/grader 不合格 run,诊断模式可显式纳入", () => {
+  const runs = [
+    {
+      taskId: "t1", runId: "completed", version: "v1", scoredAt: "x", state: "scored",
+      leaderboardEligible: true,
+      results: [{ kind: "rubric-judge", value: { correctness: 5, completeness: 5 } }],
+    },
+    {
+      taskId: "t1", runId: "timeout", version: "v1", scoredAt: "x", state: "scored",
+      runState: "timeout", runCompleted: false, leaderboardEligible: false,
+      results: [{ kind: "rubric-judge", value: { correctness: 1, completeness: 1 } }],
+    },
+  ] as any;
+  const official = buildLeaderboard(runs, catOf, REG).find((t) => t.category === "survey-writing")!;
+  const officialCorrectness = official.rows[0].cells.find((c) => c.metric === "correctness")!;
+  assert.deepEqual(officialCorrectness.coverage, { scored: 1, total: 1 });
+  assert.equal(officialCorrectness.value, 1);
+
+  const diagnostic = buildLeaderboard(runs, catOf, REG, { includeIneligible: true })
+    .find((t) => t.category === "survey-writing")!;
+  const diagnosticCorrectness = diagnostic.rows[0].cells.find((c) => c.metric === "correctness")!;
+  assert.deepEqual(diagnosticCorrectness.coverage, { scored: 2, total: 2 });
+  assert.equal(diagnosticCorrectness.value, 0.5);
+});
+
+test("buildLeaderboard: non-lifecycle grader failures retain coverage semantics", () => {
+  const runs = [{
+    taskId: "t1", runId: "grader-failed", version: "v1", scoredAt: "x",
+    state: "scoring_failed", runCompleted: true, graderState: "scoring_failed", leaderboardEligible: false,
+    results: [],
+  }] as any;
+  const table = buildLeaderboard(runs, catOf, REG).find((t) => t.category === "survey-writing")!;
+  assert.equal(table.rows[0].states.scoring_failed, 1);
+  assert.deepEqual(table.rows[0].cells[0].coverage, { scored: 0, total: 1 });
+  assert.equal(table.rows[0].cells[0].value, null);
+});

src/leaderboard.ts

@@ -72,10 +72,15 @@ export function buildLeaderboard(
   runs: RunScores[],
   categoryOf: (taskId: string) => string | undefined,
   reg: CategoryRegistry,
+  opts: { includeIneligible?: boolean } = {},
 ): CategoryTable[] {
   // 1) 按 category 分组 runs。
   const byCat = new Map<string, RunScores[]>();
   for (const run of runs) {
+    // Official tables exclude explicitly incomplete lifecycle attempts. Valid
+    // completed attempts whose grader failed remain visible in the coverage
+    // denominator/state counts, preserving the benchmark's unscored semantics.
+    if (!opts.includeIneligible && run.runCompleted === false) continue;
     const cat = categoryOf(run.taskId);
     if (!cat) continue; // 无 category 的 task 不进分类表
     const arr = byCat.get(cat);

src/score.test.ts

@@ -3,7 +3,7 @@ import assert from "node:assert/strict";
 import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
-import { runScorers, bundleWorkspaceFiles, type RunBundle } from "./score.js";
+import { classifyRunScores, lifecycleFromSignals, runScorers, bundleWorkspaceFiles, type RunBundle } from "./score.js";
 import { registerScorer } from "./scorer/registry.js";
 import type { Task } from "./task.js";
 
@@ -113,3 +113,41 @@ test("runScorers: discards results if a scorer mutates submitted telemetry", asy
     await assert.rejects(() => runScorers(fakeTask(["test-mutate-telemetry"]), bundle, "ts"), /changed during scoring/);
   } finally { rmSync(dir, { recursive: true, force: true }); }
 });
+
+test("lifecycleFromSignals: preserves explicit terminal reasons and tolerates external bundles", () => {
+  assert.deepEqual(lifecycleFromSignals({ completed: false, reason: "timeout" }), { state: "timeout", completed: false });
+  assert.deepEqual(lifecycleFromSignals({ completed: false }), { state: "incomplete", completed: false });
+  assert.deepEqual(lifecycleFromSignals({ completed: true, reason: "completed" }), { state: "completed", completed: true });
+  assert.deepEqual(lifecycleFromSignals(undefined), { state: "unknown", completed: null });
+});
+
+test("classifyRunScores: valid timed-out artifacts keep grader scores but are leaderboard-ineligible", () => {
+  const classified = classifyRunScores({
+    taskId: "t1", runId: "r", version: "v", scoredAt: "ts", state: "scored",
+    results: [{ kind: "exec-script", value: { score: 0.8 } }],
+  }, "valid", { completed: false, reason: "timeout" });
+  assert.equal(classified.state, "scored");
+  assert.equal(classified.artifactState, "valid");
+  assert.equal(classified.runState, "timeout");
+  assert.equal(classified.graderState, "scored");
+  assert.equal(classified.leaderboardEligible, false);
+  assert.equal(classified.results[0].value && (classified.results[0].value as any).score, 0.8);
+});
+
+test("classifyRunScores: invalid artifacts record that the grader did not run", () => {
+  const classified = classifyRunScores({
+    taskId: "t1", runId: "r", version: "v", scoredAt: "ts", state: "submission_invalid", results: [],
+  }, "invalid", { completed: false, reason: "error" });
+  assert.equal(classified.artifactState, "invalid");
+  assert.equal(classified.graderState, "not_run");
+  assert.equal(classified.leaderboardEligible, false);
+});
+
+test("classifyRunScores: completed valid grader failures are independently ineligible", () => {
+  const classified = classifyRunScores({
+    taskId: "t1", runId: "r", version: "v", scoredAt: "ts", state: "scoring_failed", results: [],
+  }, "valid", { completed: true, reason: "completed" });
+  assert.equal(classified.graderState, "scoring_failed");
+  assert.equal(classified.leaderboardEligible, false);
+  assert.equal(classified.leaderboardExclusionReason, "grader state is scoring_failed");
+});