ct-runner: only reports deselected rows; capability wins over selection (#286)

README.md

@@ -21,7 +21,7 @@ Pre-1.0, but densely gated:
   across all directories (remaining: named xfail classes — deferred threads
   ([#12](https://github.com/polymorph-components/polyengine/issues/12)),
   cm705-sync-sched ([#249](https://github.com/polymorph-components/polyengine/issues/249)),
-  cm705-reentrance ([#279](https://github.com/polymorph-components/polyengine/issues/279)),
+  cm707-cancel ([#250](https://github.com/polymorph-components/polyengine/issues/250)),
   upstream-nyi ([#248](https://github.com/polymorph-components/polyengine/issues/248))),
   identical on Deno, Chromium, and Firefox
   (behind its JSPI pref); WebKit reaches the same totals on trunk builds

ct-runner/src/main.ts

@@ -123,7 +123,7 @@
 
 async function loadImportsModule(path: string): Promise<Record<string, unknown>> {
   const mod = await import(
     path.startsWith(".") || path.startsWith("/")
       ? new URL(path, `file://${Deno.cwd()}/`).href
       : path
   );
@@ -212,7 +212,7 @@
     await Deno.writeTextFile(cli.out, lines.join("\n") + "\n");
     console.error(
       `${counts.passed} passed | ${counts.failed} failed | ${counts.skipped} skipped | ` +
-        `${counts.na} n/a (${counts.total} total) -> ${cli.out}`,
+        `${counts.na} n/a | ${counts.deselected} deselected (${counts.total} total) -> ${cli.out}`,
     );
     if (counts.failed > 0) Deno.exit(1);
   } catch (e) {

ct-runner/src/run-suite.ts

@@ -52,8 +52,15 @@ export interface RunSuiteOptions {
    * does the same normalization (`replaceAll("-", "_")`).
    */
   suiteName: string;
-  /** Substring filter: non-matching cases are skipped entirely (no emit),
-   * per js/viewer/harness.mjs `runCases`'s `only` handling. */
+  /** Substring selection: census cases outside it are reported `deselected`
+   * (never executed) rather than omitted, so subset runs keep full
+   * coverage with the subsetting visible as selection policy
+   * (js/viewer/harness.mjs `runCases`'s `only` doc comment;
+   * docs/runner-policy.md "Selection is not capability"). Capability wins:
+   * a tags-excluded case stays `not-applicable` even outside the
+   * selection. A filter matching no census case throws (empty selection is
+   * a run error) when the loop sees the whole census — unsharded; pooled
+   * coordinators apply the same guard over merged counts. */
   only?: string;
   /**
    * Feature-tag scheduling (issue #25): the features this target LACKS —
@@ -98,7 +105,9 @@ export interface RunSuiteOptions {
    * nor emitted, exactly as if it never existed for this shard) — this is
    * the interpretation that keeps the invariant "the union of every shard's
    * rows, in suite order, equals the unsharded run's rows" (pinned by
-   * shard_test.ts's partition-identity test).
+   * shard_test.ts's partition-identity test). Cases IN the stripe that are
+   * then filtered out by `only` still get their `deselected` row, same as
+   * an unsharded run.
    *
    * Sharded envelope/terminator contract: a sharded call still emits its
    * own envelope line and its own `{"segment-end":true}` terminator —
@@ -133,6 +142,12 @@ export interface RunCounts {
   skipped: number;
   /** Cases scheduled out as `not-applicable` (tag gating; harness.mjs `na`). */
   na: number;
+  /** Cases outside `only` (harness.mjs `deselected`): never executed, but
+   * emitted as a `deselected` row (capability outranks selection). */
+  deselected: number;
+  /** Census cases matching the selection (all of them without `only`),
+   * regardless of applicability (harness.mjs `runCases` doc comment). */
+  selected: number;
   total: number;
 }
 
@@ -170,9 +185,10 @@ function describeThrow(e: unknown): string {
  * emit the complete results-JSONL stream (envelope, one line per case,
  * terminator) through `opts.emit`. Throws `MissingImportsError` up front
  * (contracts/embedder-api.md's `requiredImports`) if the caller's imports
- * cannot satisfy the suite, and a plain `Error` if the census is empty (an
- * empty selection is a run error, per component-test-results/src/lib.rs's
- * `fold_jsonl` and harness.mjs's `runSuiteJsonl` — both refuse it).
+ * cannot satisfy the suite, and a plain `Error` if the census is empty or
+ * an unsharded `only` selects nothing (an empty selection is a run error,
+ * per component-test-results/src/lib.rs's `fold_jsonl` and harness.mjs's
+ * `runSuiteJsonl`/`runCases` — all refuse it).
  */
 export async function runSuite(
   artifacts: ComponentArtifacts,
@@ -256,7 +272,15 @@ export async function runSuite(
     );
   }
 
-  const counts: RunCounts = { passed: 0, failed: 0, skipped: 0, na: 0, total: 0 };
+  const counts: RunCounts = {
+    passed: 0,
+    failed: 0,
+    skipped: 0,
+    na: 0,
+    deselected: 0,
+    selected: 0,
+    total: 0,
+  };
 
   for (const [i, testCase] of census.entries()) {
     // Stripe membership (issue #110) is decided on the census index `i`,
@@ -267,13 +291,17 @@ export async function runSuite(
 
     const name = String(await testCase.name());
     counts.total++;
-    // js/viewer/harness.mjs `runCases`: "if (only && !name.includes(only))
-    // continue" — a filtered-out case is skipped entirely, no emit.
-    if (opts.only && !name.includes(opts.only)) continue;
+    // harness.mjs `runCases`: `isSelected` is computed up front (before tag
+    // gating) and counted in `selected` regardless of applicability — a
+    // case can be both selected and N/A.
+    const isSelected = !opts.only || name.includes(opts.only);
+    if (isSelected) counts.selected++;
 
-    // harness.mjs `runCases` mark scheduling, in its exact order: `only`
-    // first (above), then drift, then applicability. The N/A row's shape is
-    // the embed runner's (expected/verify-pipeline-fixture.jsonl):
+    // harness.mjs `runCases` mark scheduling, in its exact order:
+    // applicability first, THEN selection — "capability wins over
+    // selection" (docs/runner-policy.md "Selection is not capability"): a
+    // tags-excluded case is N/A regardless of `only`. The N/A row's shape
+    // is the embed runner's (expected/verify-pipeline-fixture.jsonl):
     // status, first excluding mark as detail, diagnostics-complete true.
     if (inventory !== null) {
       const tags = tagsOf(inventory, name);
@@ -293,6 +321,23 @@ export async function runSuite(
       }
     }
 
+    // A case that applies but sits outside `only`: reported `deselected`
+    // (never executed) rather than omitted, so subset runs keep full
+    // coverage with the subsetting visible as selection policy
+    // (harness.mjs `runCases`; docs/runner-policy.md "Selection is not
+    // capability"). Exact row shape (harness.mjs:197): case, status,
+    // `only <filter>` detail — no other fields.
+    if (!isSelected) {
+      counts.deselected++;
+      opts.emit(JSON.stringify({
+        case: name,
+        status: "deselected",
+        detail: `only ${opts.only}`,
+      }), i);
+      opts.log?.(`${name} … deselected`);
+      continue;
+    }
+
     // js/viewer/harness.mjs `runCases`' `freshCases` branch: re-enumerate
     // from a fresh instance and run the matching case; a vanished case is
     // inventory drift, not a failing case, and throws.
@@ -419,6 +464,17 @@ export async function runSuite(
     opts.log?.(`${name} … ${event.status}`);
   }
 
+  // The reference runner's empty-selection rule (a typo'd filter must not
+  // exit green with the whole census deselected), applied where the whole
+  // census is visible — unsharded (harness.mjs `runCases`: "sharded stripes
+  // may legitimately match nothing; their coordinator guards over merged
+  // counts").
+  if (opts.only && opts.shard === undefined && counts.selected === 0) {
+    throw new Error(
+      `only \`${opts.only}\` matches no cases (empty selection is a run error)`,
+    );
+  }
+
   opts.emit('{"segment-end":true}');
   return counts;
 }

ct-runner/tests/e2e_test.ts

@@ -1,7 +1,8 @@
 // Integration: the fixture suite end to end through `runSuite`, asserting
 // the case-loop policy mirrored from js/viewer/harness.mjs `runCases`/
 // `runSuiteJsonl`: fresh instance per case (default), diagnostics attached
-// to the right case, `only` filtering skips without emitting, and counts.
+// to the right case, `only` reports the unselected remainder as
+// `deselected` rows (never omitted), and counts.
 
 import { assertEq } from "../../runtime/tests/support/asserts.ts";
 import { runSuite } from "../src/mod.ts";
@@ -20,7 +21,15 @@ Deno.test({
       suiteName: "test-suite",
       emit: (l) => lines.push(l),
     });
-    assertEq(counts, { passed: 4, failed: 1, skipped: 1, na: 0, total: 6 });
+    assertEq(counts, {
+      passed: 4,
+      failed: 1,
+      skipped: 1,
+      na: 0,
+      deselected: 0,
+      selected: 6,
+      total: 6,
+    });
     assertEq(lines.length, 1 + 6 + 1); // envelope + 6 cases + terminator
 
     const events = lines.slice(1, -1).map((l) => JSON.parse(l));
@@ -64,7 +73,7 @@ Deno.test({
 });
 
 Deno.test({
-  name: "e2e: `only` filters cases out of the run entirely (no emit)",
+  name: "e2e: `only` reports the unselected remainder as `deselected` rows",
   ignore: !ready,
   fn: async () => {
     const artifacts = await artifactsOf(TEST_SUITE_WASM);
@@ -75,10 +84,65 @@ Deno.test({
       only: "diag/",
       emit: (l) => lines.push(l),
     });
-    const cases = lines.slice(1, -1).map((l) => JSON.parse(l).case);
-    assertEq(cases, ["suite/diag/chatty", "suite/diag/slow"]);
-    assertEq(counts.total, 6, "total counts every enumerated case, filtered or not");
-    assertEq(counts.passed, 2);
+    const rows = lines.slice(1, -1).map((l) => JSON.parse(l));
+    assertEq(rows.length, 6, "every census case still gets a row");
+    assertEq(rows.map((r) => r.case), [
+      "suite/basic/pass",
+      "suite/basic/fail",
+      "suite/basic/skip",
+      "suite/diag/chatty",
+      "suite/diag/slow",
+      "suite/nested/deep/leaf",
+    ]);
+    // The two `diag/` cases execute normally; the rest are exactly the
+    // `deselected` row shape (harness.mjs:197): case, status, detail — no
+    // other fields.
+    const byCase = Object.fromEntries(rows.map((r) => [r.case, r]));
+    for (
+      const name of [
+        "suite/basic/pass",
+        "suite/basic/fail",
+        "suite/basic/skip",
+        "suite/nested/deep/leaf",
+      ]
+    ) {
+      assertEq(byCase[name], {
+        case: name,
+        status: "deselected",
+        detail: "only diag/",
+      });
+    }
+    assertEq(byCase["suite/diag/chatty"].status, "pass");
+    assertEq(byCase["suite/diag/slow"].status, "pass");
+    assertEq(counts, {
+      passed: 2,
+      failed: 0,
+      skipped: 0,
+      na: 0,
+      deselected: 4,
+      selected: 2,
+      total: 6,
+    });
+  },
+});
+
+Deno.test({
+  name: "e2e: `only` matching nothing is a run error (unsharded)",
+  ignore: !ready,
+  fn: async () => {
+    const artifacts = await artifactsOf(TEST_SUITE_WASM);
+    let threw = "";
+    try {
+      await runSuite(artifacts, {
+        target: "t",
+        suiteName: "test-suite",
+        only: "no-such-case",
+        emit: () => {},
+      });
+    } catch (e) {
+      threw = String(e);
+    }
+    assertEq(threw.includes("matches no cases"), true);
   },
 });
 
@@ -94,6 +158,14 @@ Deno.test({
       freshCases: false,
       emit: (l) => lines.push(l),
     });
-    assertEq(counts, { passed: 4, failed: 1, skipped: 1, na: 0, total: 6 });
+    assertEq(counts, {
+      passed: 4,
+      failed: 1,
+      skipped: 1,
+      na: 0,
+      deselected: 0,
+      selected: 6,
+      total: 6,
+    });
   },
 });

ct-runner/tests/golden_test.ts

@@ -33,7 +33,15 @@ Deno.test({
       suiteName: "test-suite",
       emit: (l) => lines.push(l),
     });
-    assertEq(counts, { passed: 4, failed: 1, skipped: 1, na: 0, total: 6 });
+    assertEq(counts, {
+      passed: 4,
+      failed: 1,
+      skipped: 1,
+      na: 0,
+      deselected: 0,
+      selected: 6,
+      total: 6,
+    });
 
     const got = lines.map(normalize).join("\n") + "\n";
     const want = await Deno.readTextFile(

ct-runner/tests/shard_test.ts

@@ -199,7 +199,15 @@ Deno.test({
       suiteName: "test-suite",
       emit: (l) => lines.push(l),
     });
-    assertEq(counts, { passed: 4, failed: 1, skipped: 1, na: 0, total: 6 });
+    assertEq(counts, {
+      passed: 4,
+      failed: 1,
+      skipped: 1,
+      na: 0,
+      deselected: 0,
+      selected: 6,
+      total: 6,
+    });
     assertEq(lines.length, 8); // envelope + 6 cases + terminator
   },
 });