fix(oracle-gaps): supervise the check on win32; log each CLI run in the e2e driver

.github/workflows/pr.yml

@@ -288,6 +288,44 @@ jobs:
         env:
           SHARD: ${{ matrix.shard }}
         run: pnpm exec vitest --run --config oracle-gaps/vitest.config.mjs gap-report-oracle -t "$(node scripts/pr-shards.mjs pattern oracle-gaps-fixed-seed-windows "$SHARD")"
+  daemon-history-runs:
+    # Temporary diagnostic (the per-run log of the daemon history); remove before merge.
+    name: Daemon history run log (${{ matrix.os }})
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 45
+    continue-on-error: true
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
+      - name: Setup Tools
+        uses: tanstack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3
+      - run: pnpm --filter @tanstack/oracles run build
+      - name: Daemon history, with its run log
+        shell: bash
+        working-directory: packages/oracles
+        env:
+          GAP_RUN_LOG: ${{ runner.temp }}/runs.jsonl
+        run: |
+          start=$(date +%s)
+          pnpm exec vitest --run --testTimeout=2700000 --reporter=verbose --config oracle-gaps/vitest.config.mjs gap-report-oracle -t 'fixed seed 20261105: an edit sequence through the daemon$' || true
+          echo "wall $(( $(date +%s) - start )) s"
+          node -e '
+            const fs = require("fs")
+            const rows = fs.readFileSync(process.argv[1], "utf8").trim().split("\n").map((l) => JSON.parse(l))
+            const total = rows.reduce((s, r) => s + r.ms, 0)
+            console.log("runs logged: " + rows.length + ", total ms " + total)
+            console.log("killed by the driver timeout: " + rows.filter((r) => r.killedByTimeout).length)
+            for (const r of rows.filter((r) => r.ms > 20000)) console.log("long run: " + JSON.stringify(r))
+            const by = {}
+            for (const r of rows) { const k = r.via + " status " + r.status + " signal " + r.signal; by[k] = (by[k] ?? 0) + 1 }
+            console.log(JSON.stringify(by, null, 1))
+          ' "$GAP_RUN_LOG"
   windows-gap-cli:
     name: Windows gap CLI on a fixture
     runs-on: windows-latest

packages/oracles/oracle-gaps/gap-report-model-oracle.mjs

@@ -4257,8 +4257,7 @@ function baselineOf(plan) {
     // baseline too.
     const own =
       plan.tests.filter((t) => t.file === file).length +
-      (file === `a` && plan.consoleOutput ? 1 : 0) +
-      (file === `a` && plan.aborts ? 1 : 0)
+      (file === `a` && plan.consoleOutput ? 1 : 0)
     const broken = failing.find((f) => f.file === file)
     // The failing owner's suite holds two tests and a third of its name.
     const extra = broken?.test === undefined ? 0 : 3
@@ -4892,16 +4891,8 @@ function oracleTestFile(plan, file) {
       `})`,
       `test('fixed seed 999', () => {})`,
     )
-  // A test that ends the run's process by SIGABRT when the driver asks
-  // (`GAP_ABORT`), as a V8 out-of-memory abort does: in worker threads its
-  // process is the run's, and `process.abort()` is unavailable there. It
-  // runs and passes in every other run.
-  if (file === `a` && plan.aborts)
-    out.push(
-      `test('aborts the process when asked', () => {`,
-      `  if (process.env.GAP_ABORT === '1') process.kill(process.pid, 'SIGABRT')`,
-      `})`,
-    )
+  // The abort of an "aborts" run comes from the CLI's own check process
+  // (ORACLE_GAPS_TEST_ABORT_AT), so no test of file a is needed for it.
   // A file with no tests still loads, so the CLI can read what it owns.
   if (
     !plan.tests.some((t) => t.file === file) &&

packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs

@@ -479,6 +479,7 @@
  */
 import { spawn, spawnSync } from 'node:child_process'
 import {
+  appendFileSync,
   cpSync,
   existsSync,
   mkdirSync,
@@ -491,7 +492,7 @@ import {
   writeFileSync,
 } from 'node:fs'
 import { tmpdir } from 'node:os'
-import { dirname, join, relative, resolve } from 'node:path'
+import { basename, dirname, join, relative, resolve } from 'node:path'
 import { fileURLToPath, pathToFileURL } from 'node:url'
 import fc from 'fast-check'
 import { socketOf, stateDirOf } from './daemon.mjs'
@@ -653,18 +654,36 @@ const configArgs = (plan) => ({
  * Writes the fixture's files and runs the CLI on it. A file of an earlier
  * state that this state lacks is removed.
  */
+// Per-run log (GAP_RUN_LOG, a file the driver appends to): each CLI run's
+// start and end, its status or signal, and whether the driver's kill timeout
+// ended it. Test-side only; the CLI never sees it.
+const runLog = process.env.GAP_RUN_LOG
+const logRun = (entry) => {
+  if (runLog !== undefined) appendFileSync(runLog, `${JSON.stringify(entry)}\n`)
+}
+
 function runCli(dir, files, how = {}) {
   writeFixture(dir, files)
   const before = new Set(filesUnder(dir))
   const { args, out } = cliArgs(dir, files, how)
   if (how.logBlocked) mkdirSync(join(out, `stream.log`), { recursive: true })
+  const started = Date.now()
   const child = spawnSync(process.execPath, args, {
     cwd: repo,
     encoding: `utf8`,
     timeout: 300_000,
     maxBuffer: 64 * 1024 * 1024,
     env: cleanEnv(),
   })
+  logRun({
+    via: `runCli`,
+    dir: basename(dir),
+    ms: Date.now() - started,
+    status: child.status,
+    signal: child.signal,
+    killedByTimeout: child.error?.code === `ETIMEDOUT`,
+    daemon: how.daemon === true,
+  })
   const run = readRun(
     dir,
     out,
@@ -730,11 +749,10 @@ async function stoppedRun(
   const atSpawn = when === `spawn`
   const after = when === `after`
   const warm = when === `warm`
-  // With `abort`, no signal comes: a test of file a aborts the run's
-  // process (`GAP_ABORT`), in worker threads, so the abort ends the process
-  // that runs the check, as a V8 out-of-memory abort does. The run then
-  // ends with one error line per stream that names how its process ended,
-  // and exits 3 (the coordinator's decision of 2026-11-12 on P10).
+  // With `abort`, no signal comes: the check's process aborts in its main
+  // thread (ORACLE_GAPS_TEST_ABORT_AT, as a V8 out-of-memory abort does). The
+  // run then ends with one error line per stream that names how its process
+  // ended, and exits 3 (the coordinator's decision of 2026-11-12 on P10).
   const abort = when === `abort`
   writeFixture(dir, files)
   const before = new Set(filesUnder(dir))
@@ -765,7 +783,7 @@ async function stoppedRun(
               ORACLE_GAPS_TEST_WARM_PROBE: join(pids, `warming`),
             }
           : abort
-            ? { ...cleanEnv(), GAP_ABORT: `1` }
+            ? { ...cleanEnv(), ORACLE_GAPS_TEST_ABORT_AT: `check` }
             : { ...cleanEnv(), GAP_STOP_DIR: pids }),
     },
     stdio: [`ignore`, `pipe`, `pipe`],
@@ -1075,10 +1093,23 @@ async function runClisAtOnce(dir, files, hows) {
       child.stderr.on(`data`, (chunk) => {
         stderr += chunk
       })
+      const started = Date.now()
+      let timedOut = false
       const status = await new Promise((resolvePromise) => {
-        const timer = setTimeout(() => child.kill(), 300_000)
+        const timer = setTimeout(() => {
+          timedOut = true
+          child.kill()
+        }, 300_000)
         child.on(`close`, (code) => {
           clearTimeout(timer)
+          logRun({
+            via: `runClisAtOnce`,
+            dir: basename(dir),
+            ms: Date.now() - started,
+            status: code,
+            killedByTimeout: timedOut,
+            daemon: how.daemon === true,
+          })
           resolvePromise(code)
         })
       })
@@ -2758,7 +2789,10 @@ function check(plan, observations, expected) {
       value.when === `after`
         ? `oracle-gaps: done completed`
         : value.when === `abort`
-          ? `oracle-gaps: done error, the run's process exited with SIGABRT`
+          ? process.platform === `win32`
+            ? // win32 has no signal a process can die of: the exit code names it.
+              `oracle-gaps: done error, the run's process exited with code `
+            : `oracle-gaps: done error, the run's process exited with SIGABRT`
           : `oracle-gaps: done stopped (${value.signal})`
     if (value.spun && value.when === `warm` && value.refused)
       return violation(

packages/oracles/oracle-gaps/index.mjs

@@ -117,15 +117,15 @@
 // Vitest worker joins, and every process a test starts unless it starts it
 // detached (in a session of its own, as a browser driver can), and waits
 // for it. Help, --stop-daemon, and arguments that refuse start no process,
-// so they end in the CLI's own process, without a second Node start. Windows has no process groups, so there the run is not supervised.
+// so they end in the CLI's own process, without a second Node start. On win32, which has no process groups, the supervisor ends the child's tree with taskkill.
 // SIGINT or SIGTERM is sent on to the whole group; a process that has not
 // ended 2 s later is killed, and so is any process left in the group when
 // the run ends. The run kills its group itself if the CLI dies first.
 // The stop handlers and the claim on the output directory come before the
 // CLI's modules load (review of #57, P1): a signal during the loads is kept
 // and acted on once the stream log's note is written, so a stop as the CLI starts still
 // leaves the note and one done line. Only Node's own start is before them.
-import { spawn } from 'node:child_process'
+import { spawn, spawnSync } from 'node:child_process'
 import {
   appendFileSync,
   existsSync,
@@ -139,6 +139,7 @@ import {
 import { constants } from 'node:os'
 import { join } from 'node:path'
 import { pathToFileURL } from 'node:url'
+import { posix } from './paths.mjs'
 
 let early
 const keepEarly = (signal) => {
@@ -322,7 +323,8 @@ function startLog(out) {
     throw cannotWrite(path, error)
   }
   streamLog = path
-  const note = `oracle-gaps: note log ${path}\n`
+  // The log is named in forward slashes on every platform (the Windows paths rule).
+  const note = `oracle-gaps: note log ${posix(path)}\n`
   process.stdout.write(note)
   writeStream(note)
 }
@@ -367,7 +369,7 @@ function superviseRun(out) {
   const finish = () => {
     if (finishing) return
     finishing = true
-    if (child) signalGroup(child.pid, `SIGKILL`)
+    if (child) signalTree(child.pid, `SIGKILL`)
     let decided = false
     const decide = () => {
       if (decided) return
@@ -396,11 +398,13 @@ function superviseRun(out) {
     if (stopped) return
     stopped = signal
     if (child === undefined) return finish()
-    signalGroup(child.pid, signal)
+    // On win32 a stop after the child wrote its ending is ignored, as on POSIX,
+    // where the child ignores the signal itself.
+    if (!ending || process.platform !== `win32`) signalTree(child.pid, signal)
     const deadline = Date.now() + STOP_GRACE_MS
     const poll = setInterval(() => {
       if (
-        (ended === undefined || signalGroup(child.pid, 0)) &&
+        (ended === undefined || signalTree(child.pid, 0)) &&
         Date.now() < deadline
       )
         return
@@ -432,7 +436,9 @@ function superviseRun(out) {
       out,
     ],
     {
-      detached: true,
+      // win32 has no process groups: the child is not detached there, and its
+      // tree is ended by signalTree.
+      detached: process.platform !== `win32`,
       stdio: [`inherit`, `inherit`, `pipe`, `ipc`],
       env: { ...process.env, ORACLE_GAPS_GROUP: String(process.pid) },
     },
@@ -459,13 +465,27 @@ function superviseRun(out) {
 }
 
 /** Signals every process of group `group`; false when none is left. */
-function signalGroup(group, signal) {
-  try {
-    process.kill(-group, signal)
-    return true
-  } catch {
-    return false
+/**
+ * Signals the process tree of `pid`: its process group on POSIX, where the
+ * run's workers are members. win32 has no groups and no signals a Node process
+ * can receive, so there the tree is ended with taskkill; signal 0 asks whether
+ * the process runs.
+ */
+function signalTree(pid, signal) {
+  if (process.platform !== `win32`) {
+    try {
+      process.kill(-pid, signal)
+      return true
+    } catch {
+      return false
+    }
   }
+  if (signal === 0) return alive(pid)
+  return (
+    spawnSync(`taskkill`, [`/T`, `/F`, `/PID`, String(pid)], {
+      stdio: `ignore`,
+    }).status === 0
+  )
 }
 
 /**
@@ -474,7 +494,7 @@ function signalGroup(group, signal) {
  */
 function watchSupervisor(supervisor) {
   setInterval(() => {
-    if (process.ppid !== supervisor) process.kill(-process.pid, `SIGKILL`)
+    if (!alive(supervisor)) signalTree(process.pid, `SIGKILL`)
   }, 1000).unref()
 }
 
@@ -497,6 +517,16 @@ async function checkCold(options) {
   const { openSession } = await import(`./check.mjs`)
   const { memoryStore } = await import(`./evidence.mjs`)
   const session = await openSession(options.pkg)
+  // A test-only abort (ORACLE_GAPS_TEST_ABORT_AT=check, inert unless set), after
+  // the session has begun, where a V8 out-of-memory abort would strike; the
+  // supervisor reports how the process ended.
+  if (process.env.ORACLE_GAPS_TEST_ABORT_AT === `check`) {
+    // process.abort() prints a native backtrace on POSIX, which is no line of
+    // the stream (law 15); a SIGABRT to itself ends the process the same way
+    // without it. win32 has no such signal, so there it is process.abort().
+    if (process.platform === `win32`) process.abort()
+    else process.kill(process.pid, `SIGABRT`)
+  }
   try {
     return await session.check(options, memoryStore(), (line) =>
       writeStream(`${line}\n`),
@@ -541,25 +571,13 @@ function endWith(error, out) {
 // compared with the URL of the path it was started as.
 const main =
   import.meta.url === pathToFileURL(realpathSync(process.argv[1])).href
-// Windows has no process groups: there the run is the CLI's own process.
-const supervised = process.platform !== `win32`
 const supervisor = process.env.ORACLE_GAPS_GROUP
 
-// Set once the note is written (the unsupervised run has started).
-let runStarted = false
-/** Ends an unsupervised run with its stopped line on both streams (#58). */
-function stopNow(signal) {
-  const line = `oracle-gaps: done stopped (${signal})\n`
-  process.stdout.write(line)
-  writeStream(line)
-  exitWhenFlushed(128 + constants.signals[signal])
-}
-
 async function run() {
   // The supervisor: it claims the output directory, writes the note, and runs
   // the child. A refusal of the claim or the log ends it through endWith (the
   // single path for every refusal of a check).
-  if (supervised && supervisor === undefined) {
+  if (supervisor === undefined) {
     const checking = asksForCheck(process.argv.slice(2))
     if (checking !== undefined) {
       try {
@@ -577,25 +595,16 @@ async function run() {
   // is no refusal, and the supervisor writes the stopped line. Once the run
   // writes its ending, a stop no longer cuts it off.
   // The all-or-nothing stop law (#57 and #58): a stop after the note writes one
-  // done stopped line on each stream; a supervised child's stop is the
-  // supervisor's line; a stop before the note waits for it (the early stop).
+  // done stopped line on each stream, and the child's stop is the supervisor's
+  // line.
   const onStop = (signal) => {
     if (writingEnding) return
-    if (!supervised) return runStarted && stopNow(signal)
     process.exit(128 + constants.signals[signal])
   }
   process.on(`SIGINT`, onStop)
   process.on(`SIGTERM`, onStop)
-  // A test-only stop for unsupervised runs (ORACLE_GAPS_TEST_STOP, #58).
-  if (process.env.ORACLE_GAPS_TEST_STOP !== undefined && !supervised) {
-    const file = process.env.ORACLE_GAPS_TEST_STOP
-    setInterval(() => {
-      if (existsSync(file)) onStop(`SIGINT`)
-    }, 50).unref()
-  }
-  // A child of a supervisor that was stopped before its own handlers ran exits
-  // now (the supervisor writes the line). An unsupervised run keeps the stop
-  // and takes it once its note is written, below.
+  // A child that was stopped before its own handlers ran exits now (the
+  // supervisor writes the line).
   if (supervisor !== undefined && early !== undefined)
     process.exit(128 + constants.signals[early])
   // The run's options, once its arguments parse.
@@ -611,11 +620,6 @@ async function run() {
       const { stopDaemon } = await import(`./daemon.mjs`)
       await stopDaemon(options.pkg)
     } else {
-      // Windows has no supervisor: this process opens the log and runs the
-      // check. A child of a supervisor writes its stderr alone.
-      if (supervisor === undefined) startLog(options.out)
-      runStarted = true
-      if (early !== undefined) return stopNow(early)
       const summary = await (options.daemon ? checkThroughDaemon : checkCold)(
         options,
       )