feat: support live replay test reporters

src/__tests__/cli-capture.ts

@@ -2,7 +2,11 @@ import fs from 'node:fs';
 import os from 'node:os';
 import path from 'node:path';
 import { runCli } from '../cli.ts';
-import type { DaemonRequest, DaemonResponse } from '../daemon/client/daemon-client.ts';
+import type {
+  DaemonRequest,
+  DaemonResponse,
+  sendToDaemon,
+} from '../daemon/client/daemon-client.ts';
 import { installIsolatedCliTestEnv } from './cli-test-env.ts';
 
 class ExitSignal extends Error {
@@ -15,6 +19,7 @@ class ExitSignal extends Error {
 }
 
 export type CapturedDaemonRequest = Omit<DaemonRequest, 'token'>;
+type DaemonTransportOptions = Parameters<typeof sendToDaemon>[1];
 
 export type CapturedCliRun = {
   code: number | null;
@@ -28,11 +33,17 @@ export type CliCaptureOptions = {
   env?: Record<string, string | undefined>;
   stateDirPrefix?: string;
   passthroughBufferWrites?: boolean;
-  sendToDaemon?: (req: CapturedDaemonRequest) => Promise<DaemonResponse>;
+  sendToDaemon?: (
+    req: CapturedDaemonRequest,
+    options?: DaemonTransportOptions,
+  ) => Promise<DaemonResponse>;
   defaultResponse?: DaemonResponse;
 };
 
-type CliCaptureResponder = (req: CapturedDaemonRequest) => Promise<DaemonResponse>;
+type CliCaptureResponder = (
+  req: CapturedDaemonRequest,
+  options?: DaemonTransportOptions,
+) => Promise<DaemonResponse>;
 
 export async function runCliCapture(
   argv: string[],
@@ -82,10 +93,13 @@ export async function runCliCapture(
     return true;
   }) as typeof process.stderr.write;
 
-  const sendToDaemon = async (req: CapturedDaemonRequest): Promise<DaemonResponse> => {
+  const sendToDaemon = async (
+    req: CapturedDaemonRequest,
+    daemonOptions?: DaemonTransportOptions,
+  ): Promise<DaemonResponse> => {
     calls.push(req);
     if (options.sendToDaemon) {
-      return await options.sendToDaemon(req);
+      return await options.sendToDaemon(req, daemonOptions);
     }
     return options.defaultResponse ?? { ok: true, data: {} };
   };

src/__tests__/cli-network.test.ts

@@ -282,6 +282,87 @@ test('test command --verbose omits step telemetry for passing tests without debu
   }
 });
 
+test('test command --verbose includes step telemetry in completed progress output', async () => {
+  const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'agent-device-cli-test-live-verbose-'));
+  const artifactsDir = path.join(tmpDir, 'auth-flow');
+  const attemptDir = path.join(artifactsDir, 'attempt-1');
+  await fs.mkdir(attemptDir, { recursive: true });
+  await fs.writeFile(
+    path.join(attemptDir, 'replay-timing.ndjson'),
+    [
+      {
+        type: 'replay_action_start',
+        step: 1,
+        line: 3,
+        command: '__maestroTapOn',
+        positionals: ['text="Log in"'],
+      },
+      {
+        type: 'replay_action_stop',
+        step: 1,
+        line: 3,
+        command: '__maestroTapOn',
+        ok: true,
+        durationMs: 250,
+      },
+    ]
+      .map((entry) => JSON.stringify(entry))
+      .join('\n'),
+  );
+
+  try {
+    const result = await runCliCapture(['test', './suite', '--verbose'], async (_req, options) => {
+      options?.onProgress?.({
+        type: 'replay-test',
+        file: '/tmp/auth-flow.yml',
+        title: 'Authentication flow',
+        status: 'pass',
+        index: 1,
+        total: 1,
+        durationMs: 500,
+        attempt: 1,
+        artifactsDir,
+      });
+      return {
+        ok: true,
+        data: {
+          total: 1,
+          executed: 1,
+          passed: 1,
+          failed: 0,
+          skipped: 0,
+          notRun: 0,
+          durationMs: 500,
+          failures: [],
+          tests: [
+            {
+              file: '/tmp/auth-flow.yml',
+              title: 'Authentication flow',
+              session: 'default:test:suite:1',
+              status: 'passed',
+              durationMs: 500,
+              finalAttemptDurationMs: 500,
+              attempts: 1,
+              artifactsDir,
+              replayed: 1,
+              healed: 0,
+            },
+          ],
+        },
+      };
+    });
+
+    assert.equal(result.code, null);
+    assert.equal(result.calls[0]?.meta?.debug, false);
+    assert.match(result.stderr, /✓ Authentication flow 0\.5s/);
+    assert.match(result.stderr, /steps:/);
+    assert.match(result.stderr, /tapOn "text=\\"Log in\\"" \(line 3, 0\.25s\)/);
+    assert.doesNotMatch(result.stdout, /steps:/);
+  } finally {
+    await fs.rm(tmpDir, { recursive: true, force: true });
+  }
+});
+
 test('test command --verbose omits nested passing step telemetry', async () => {
   const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'agent-device-cli-test-verbose-retry-'));
   const artifactsDir = path.join(tmpDir, 'material-top-tabs');
@@ -771,12 +852,24 @@ test('test command loads custom reporter modules', async () => {
       'utf8',
     );
 
-    const result = await runCliCapture(['test', './suite', '--reporter', reporterPath], async () =>
-      makeReplaySuiteResponse(),
+    const result = await runCliCapture(
+      ['test', './suite', '--reporter', reporterPath],
+      async (_req, options) => {
+        options?.onProgress?.({
+          type: 'replay-test',
+          file: '/tmp/01-pass.ad',
+          status: 'pass',
+          index: 1,
+          total: 1,
+          durationMs: 10,
+        });
+        return makeReplaySuiteResponse();
+      },
     );
 
-    assert.equal(result.code, null);
+    assert.equal(result.code, 1);
     assert.doesNotMatch(result.stdout, /Test summary:/);
+    assert.doesNotMatch(result.stderr, /✓ 01-pass\.ad/);
     assert.deepEqual(JSON.parse(await fs.readFile(outputPath, 'utf8')), {
       total: 3,
       failed: 1,
@@ -785,3 +878,153 @@ test('test command loads custom reporter modules', async () => {
     await fs.rm(tmpDir, { recursive: true, force: true });
   }
 });
+
+test('test command streams progress to custom reporter modules', async () => {
+  const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'agent-device-live-reporter-test-'));
+  const reporterPath = path.join(tmpDir, 'live-reporter.mjs');
+
+  try {
+    await fs.writeFile(
+      reporterPath,
+      [
+        'export default {',
+        "  name: 'live-custom',",
+        '  onTestStep(test, context) {',
+        '    context.stderr.write(`live:progress:${test.stepIndex ?? 0}/${test.stepTotal ?? 0}\\n`);',
+        '  },',
+        '  onTestResult(test, context) {',
+        '    context.stderr.write(`live:${test.status}:0/0\\n`);',
+        '  },',
+        '  onSuiteEnd(suite, context) {',
+        '    context.stdout.write(`final:${suite.total}\\n`);',
+        '  },',
+        '  getExitCode() { return 0; },',
+        '};',
+      ].join('\n'),
+      'utf8',
+    );
+
+    const result = await runCliCapture(
+      ['test', './suite', '--reporter', reporterPath],
+      async (_req, options) => {
+        options?.onProgress?.({
+          type: 'replay-test',
+          file: '/tmp/01-pass.ad',
+          status: 'progress',
+          index: 1,
+          total: 1,
+          stepIndex: 1,
+          stepTotal: 2,
+        });
+        options?.onProgress?.({
+          type: 'replay-test',
+          file: '/tmp/01-pass.ad',
+          status: 'pass',
+          index: 1,
+          total: 1,
+          durationMs: 10,
+        });
+        return makeReplaySuiteResponse();
+      },
+    );
+
+    assert.equal(result.code, 1);
+    assert.equal(result.stdout, 'final:3\n');
+    assert.match(result.stderr, /live:progress:1\/2/);
+    assert.match(result.stderr, /live:pass:0\/0/);
+  } finally {
+    await fs.rm(tmpDir, { recursive: true, force: true });
+  }
+});
+
+test('test command reuses custom reporter instance for progress and final output', async () => {
+  const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'agent-device-stateful-reporter-test-'));
+  const reporterPath = path.join(tmpDir, 'stateful-reporter.mjs');
+
+  try {
+    await fs.writeFile(
+      reporterPath,
+      [
+        'export default function createReporter() {',
+        '  const seen = [];',
+        '  return {',
+        "    name: 'stateful-custom',",
+        '    onTestResult(test) {',
+        '      seen.push(test.status);',
+        '    },',
+        '    onSuiteEnd(_suite, context) {',
+        '      context.stdout.write(`seen:${seen.join(",")}\\n`);',
+        '    },',
+        '    getExitCode() { return 0; },',
+        '  };',
+        '}',
+      ].join('\n'),
+      'utf8',
+    );
+
+    const result = await runCliCapture(
+      ['test', './suite', '--reporter', reporterPath],
+      async (_req, options) => {
+        options?.onProgress?.({
+          type: 'replay-test',
+          file: '/tmp/01-pass.ad',
+          status: 'pass',
+          index: 1,
+          total: 1,
+          durationMs: 10,
+        });
+        return makeReplaySuiteResponse();
+      },
+    );
+
+    assert.equal(result.code, 1);
+    assert.equal(result.stdout, 'seen:pass\n');
+  } finally {
+    await fs.rm(tmpDir, { recursive: true, force: true });
+  }
+});
+
+test('test command surfaces a throwing live reporter hook without aborting the run', async () => {
+  const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'agent-device-throwing-reporter-test-'));
+  const reporterPath = path.join(tmpDir, 'throwing-reporter.mjs');
+
+  try {
+    await fs.writeFile(
+      reporterPath,
+      [
+        'export default {',
+        "  name: 'throwing-custom',",
+        '  onTestResult() {',
+        "    throw new Error('boom');",
+        '  },',
+        '  onSuiteEnd(suite, context) {',
+        '    context.stdout.write(`final:${suite.total}\\n`);',
+        '  },',
+        '  getExitCode() { return 0; },',
+        '};',
+      ].join('\n'),
+      'utf8',
+    );
+
+    const result = await runCliCapture(
+      ['test', './suite', '--reporter', reporterPath],
+      async (_req, options) => {
+        options?.onProgress?.({
+          type: 'replay-test',
+          file: '/tmp/01-pass.ad',
+          status: 'pass',
+          index: 1,
+          total: 1,
+          durationMs: 10,
+        });
+        return makeReplaySuiteResponse();
+      },
+    );
+
+    assert.equal(result.code, 1);
+    assert.equal(result.stdout, 'final:3\n');
+    assert.match(result.stderr, /Reporter throwing-custom onTestResult failed: boom/);
+  } finally {
+    await fs.rm(tmpDir, { recursive: true, force: true });
+  }
+});