Harden Codex CUA facade isolation, evidence, and tool logging

packages/evals/framework/codexRunner.ts

@@ -132,6 +132,11 @@ export async function runCodexAgent({
         },
         outputSchema: EVAL_RESULT_SCHEMA,
         maxToolSteps,
+        diagnosticDirectory: process.env.EVAL_CODEX_DIAGNOSTICS_DIR,
+        allowedMcpServers:
+          toolAdapter && "allowedMcpServers" in toolAdapter
+            ? toolAdapter.allowedMcpServers
+            : undefined,
         ...(toolAdapter?.env?.CODEX_HOME && { codexHome: toolAdapter.env.CODEX_HOME }),
         onToolStep:
           toolAdapter && "recordObservation" in toolAdapter

packages/evals/framework/codexToolAdapter.ts

@@ -1,6 +1,7 @@
 import fsp from "node:fs/promises";
 import os from "node:os";
 import path from "node:path";
+import { isolatedCodexEnv } from "@browserbasehq/stagehand-integrations-codex-sdk";
 import { EvalsError } from "../errors.js";
 import type { EvalLogger } from "../logger.js";
 import {
@@ -51,7 +52,8 @@ export interface PreparedCodexCodeAdapter {
    * it to record per-step observations (their tool calls never pass through
    * the workspace bridge).
    */
-  recordObservation?: () => void;
+  recordObservation?: (item: Record<string, unknown>) => Promise<void>;
+  allowedMcpServers?: string[];
   /** Which normalized tool-call names consume observation indexes. */
   observedToolMatcher?: (name: string) => boolean;
   cleanup: () => Promise<void>;
@@ -85,7 +87,7 @@ const STAGEHAND_FACADE_MCP_TIMEOUTS = {
 export const CODEX_MCP_TOOLS_APPROVAL_MODE = "approve";
 
 /** Name of the per-run Codex home directory created inside the adapter cwd. */
-export const CODEX_HOME_DIRNAME = ".codex-home";
+export const CODEX_HOME_DIRNAME = path.join("home", ".codex");
 
 export function buildCodexMcpServers(
   toolSurface: ToolSurface,
@@ -116,15 +118,16 @@ export function buildIsolatedCodexEnv(
 ): Record<string, string> {
   const env: Record<string, string> = {};
   for (const [key, value] of Object.entries(baseEnv)) {
-    if (value !== undefined) env[key] = value;
+    if (value !== undefined && (!key.startsWith("CODEX_") || key === "CODEX_API_KEY"))
+      env[key] = value;
   }
+  env.HOME = path.dirname(codexHome);
   env.CODEX_HOME = codexHome;
   return env;
 }
 
 async function createIsolatedCodexHome(cwd: string): Promise<string> {
-  const codexHome = path.join(cwd, CODEX_HOME_DIRNAME);
-  await fsp.mkdir(codexHome, { recursive: true });
+  const { CODEX_HOME: codexHome } = await isolatedCodexEnv(cwd);
   // Every setting the run needs arrives as `--config` overrides from the SDK;
   // the file exists only so nothing in this home is inherited from elsewhere.
   await fsp.writeFile(
@@ -248,6 +251,7 @@ export async function prepareCodexToolAdapter(
         promptInstructions: mount.promptInstructions,
         browserSession: runtime.browserSession,
         codexConfig: { mcp_servers: codexMcpServers },
+        allowedMcpServers: serverNames,
         ...(runtime.running.browserSessionLoss && {
           browserSessionLoss: runtime.running.browserSessionLoss,
         }),
@@ -259,7 +263,11 @@ export async function prepareCodexToolAdapter(
             await recorder.settle();
             return recorder.drain();
           },
-          recordObservation: () => void recorder.record(),
+          recordObservation: async (item: Record<string, unknown>) => {
+            if (serverNames.includes(String(item.server))) {
+              await recorder.record(typeof item.id === "string" ? item.id : undefined);
+            }
+          },
         }),
         observedToolMatcher: (name: string) =>
           serverNames.some((server) => name.startsWith(`${server}.`)),

packages/evals/framework/harnesses/codexAdapter.ts

@@ -85,6 +85,7 @@ export class CodexTrajectoryAdapter implements TrajectoryAdapter<CodexRunResult>
 
       const call = normalizeItem(itemType, item, pendingReasoning);
       if (call) {
+        if (typeof item.id === "string") call.id = item.id;
         toolCalls.push(call);
         pendingReasoning = "";
       }
@@ -96,7 +97,17 @@ export class CodexTrajectoryAdapter implements TrajectoryAdapter<CodexRunResult>
     // the bridge some other way), ordinals would shift and attach evidence
     // to the wrong steps — misattribution is worse than a gap, so attach
     // nothing and let the verifier take its evidence_insufficient path.
-    const observations = result.stepObservations ?? [];
+    const keyed = new Map(
+      (result.stepObservations ?? [])
+        .filter((observation) => observation.toolCallId)
+        .map((observation) => [observation.toolCallId, observation.evidence]),
+    );
+    for (const call of toolCalls) {
+      if (call.id && keyed.has(call.id)) call.probeEvidence = keyed.get(call.id);
+    }
+    const observations = (result.stepObservations ?? []).filter(
+      (observation) => !observation.toolCallId,
+    );
     if (observations.length > 0) {
       const observedCalls = toolCalls.filter((call) =>
         result.observedToolName

packages/evals/framework/harnesses/trajectoryAdapter.ts

@@ -43,6 +43,7 @@ export interface TrajectoryAdapter<THarnessResult> {
  * this shape before mapping to a TrajectoryStep.
  */
 export interface NormalizedToolCall {
+  id?: string;
   /** Tool name (e.g., "Bash", "mcp__stagehand_browser__run", "container.exec"). */
   name: string;
   /** Tool arguments. Empty object if the harness doesn't surface them. */

packages/evals/framework/observationRecorder.ts

@@ -3,6 +3,7 @@ import type { ProbeEvidence } from "stagehand-v3";
 /** A probe observation captured after the Nth run-tool execution (0-based). */
 export interface StepObservation {
   runIndex: number;
+  toolCallId?: string;
   evidence: ProbeEvidence;
 }
 
@@ -32,13 +33,13 @@ export class ObservationRecorder {
 
   constructor(private readonly capture: () => Promise<ProbeEvidence>) {}
 
-  async record(): Promise<void> {
+  async record(toolCallId?: string): Promise<void> {
     const runIndex = this.runIndex++;
     const attempt = (async () => {
       try {
         const evidence = await withTimeout(this.capture(), observationTimeoutMs());
         if (evidence.screenshot || evidence.url || evidence.ariaTree) {
-          this.observations.push({ runIndex, evidence });
+          this.observations.push({ runIndex, evidence, ...(toolCallId && { toolCallId }) });
         }
       } catch {
         // best-effort only — a failed probe must never fail the run tool

packages/evals/tests/framework/codexToolAdapter.test.ts

@@ -64,9 +64,19 @@ describe("codex tool adapter", () => {
 
   it("points CODEX_HOME at the per-run directory and drops the inherited one", () => {
     const env = buildIsolatedCodexEnv(
-      { PATH: "/usr/bin", CODEX_HOME: "/Users/someone/.codex", UNSET: undefined },
-      "/tmp/run/.codex-home",
+      {
+        PATH: "/usr/bin",
+        CODEX_HOME: "/Users/someone/.codex",
+        CODEX_THREAD_ID: "host",
+        HOME: "/Users/someone",
+        UNSET: undefined,
+      },
+      "/tmp/run/home/.codex",
     );
-    expect(env).toEqual({ PATH: "/usr/bin", CODEX_HOME: "/tmp/run/.codex-home" });
+    expect(env).toEqual({
+      PATH: "/usr/bin",
+      HOME: "/tmp/run/home",
+      CODEX_HOME: "/tmp/run/home/.codex",
+    });
   });
 });

packages/evals/tests/framework/harnessObservations.test.ts

@@ -17,6 +17,55 @@ import {
 
 const TASK_SPEC: TaskSpec = { id: "t", instruction: "do the thing" };
 
+describe("Codex keyed observations", () => {
+  it("retains early evidence despite unrelated calls and a missing final capture", () => {
+    const trajectory = codexAdapter.fromHarnessResult(
+      {
+        events: [
+          {
+            type: "item.completed",
+            item: {
+              id: "other",
+              type: "mcp_tool_call",
+              server: "other",
+              tool: "run",
+              status: "completed",
+            },
+          },
+          {
+            type: "item.completed",
+            item: {
+              id: "first",
+              type: "mcp_tool_call",
+              server: "stagehand",
+              tool: "run",
+              status: "completed",
+            },
+          },
+          {
+            type: "item.completed",
+            item: {
+              id: "last",
+              type: "mcp_tool_call",
+              server: "stagehand",
+              tool: "run",
+              status: "completed",
+            },
+          },
+        ],
+        stepObservations: [
+          { runIndex: 0, toolCallId: "first", evidence: { url: "https://example.com/first" } },
+        ],
+        observedToolName: (name) => name.startsWith("stagehand."),
+      },
+      TASK_SPEC,
+    );
+    expect(trajectory.steps[0].probeEvidence?.url).toBeUndefined();
+    expect(trajectory.steps[1].probeEvidence?.url).toBe("https://example.com/first");
+    expect(trajectory.steps[2].probeEvidence?.url).toBeUndefined();
+  });
+});
+
 describe("observation recorder", () => {
   afterEach(() => {
     delete process.env.EVAL_HARNESS_OBSERVATIONS;

packages/integrations/codex-sdk/src/diagnostics.ts

@@ -0,0 +1,40 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+import { randomUUID } from "node:crypto";
+import { sanitizeErrorMessage } from "@browserbasehq/stagehand-integrations/harness";
+
+export async function saveCodexDiagnostic(
+  directory: string,
+  error: unknown,
+  eventCount: number,
+): Promise<string> {
+  await fs.mkdir(directory, { recursive: true, mode: 0o700 });
+  const message = error instanceof Error ? error.message : String(error);
+  const cause = error instanceof Error ? error.cause : undefined;
+  const filename = path.join(directory, `${Date.now()}-${randomUUID()}.json`);
+  const sanitized = sanitizeErrorMessage(message);
+  const maxChars = 2_000_000;
+  const marker = "Failed to parse item: ";
+  const rejectedEvent = sanitized.startsWith(marker) ? sanitized.slice(marker.length) : undefined;
+  await fs.writeFile(
+    filename,
+    JSON.stringify(
+      {
+        timestamp: new Date().toISOString(),
+        eventCount,
+        message: sanitized.slice(0, maxChars),
+        truncated: sanitized.length > maxChars,
+        cause: cause ? sanitizeErrorMessage(String(cause)).slice(0, 16_000) : undefined,
+        rejectedEvent: rejectedEvent?.slice(0, maxChars),
+        stack:
+          error instanceof Error
+            ? sanitizeErrorMessage(error.stack ?? "").slice(0, 16_000)
+            : undefined,
+      },
+      null,
+      2,
+    ),
+    { mode: 0o600, flag: "wx" },
+  );
+  return filename;
+}

packages/integrations/codex-sdk/src/index.ts

@@ -1 +1,2 @@
 export * from "./session.js";
+export { isolatedCodexEnv } from "./isolation.js";

packages/integrations/codex-sdk/src/isolation.ts

@@ -0,0 +1,28 @@
+import fs from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+
+export async function isolatedCodexEnv(
+  cwd: string,
+  source: NodeJS.ProcessEnv = process.env,
+): Promise<Record<string, string>> {
+  const home = path.join(cwd, "home");
+  const codexHome = path.join(home, ".codex");
+  await fs.mkdir(codexHome, { recursive: true, mode: 0o700 });
+  const originalHome = source.CODEX_HOME ?? path.join(source.HOME ?? os.homedir(), ".codex");
+  if (!source.OPENAI_API_KEY && !source.CODEX_API_KEY) {
+    try {
+      const auth = await fs.readFile(path.join(originalHome, "auth.json"));
+      await fs.writeFile(path.join(codexHome, "auth.json"), auth, { mode: 0o600 });
+    } catch (error) {
+      if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
+    }
+  }
+  const env = Object.fromEntries(
+    Object.entries(source).filter(
+      ([key, value]) =>
+        value !== undefined && (!key.startsWith("CODEX_") || key === "CODEX_API_KEY"),
+    ),
+  ) as Record<string, string>;
+  return { ...env, HOME: home, CODEX_HOME: codexHome };
+}

packages/integrations/codex-sdk/src/session.ts

@@ -2,6 +2,7 @@ import type { Dirent } from "node:fs";
 import fsp from "node:fs/promises";
 import path from "node:path";
 import type { ModelReasoningEffort } from "@openai/codex-sdk";
+import { saveCodexDiagnostic } from "./diagnostics.js";
 import {
   HarnessAdapterError,
   harnessEventLogLevel,
@@ -58,6 +59,7 @@ export type CodexSessionResult = {
   usageSource: CodexUsageSource;
   threadId?: string;
   iterationError?: unknown;
+  diagnosticPath?: string;
 };
 
 export const CODEX_SDK_PACKAGE = "@openai/codex-sdk";
@@ -145,20 +147,23 @@ export async function runCodexSession(input: {
   thread: CodexThreadConfig;
   outputSchema?: Record<string, unknown>;
   maxToolSteps?: number;
-  onToolStep?: () => void | Promise<void>;
+  onToolStep?: (item: Record<string, unknown>) => void | Promise<void>;
+  allowedMcpServers?: string[];
   /**
    * CODEX_HOME the binary runs with. `codex exec` only reports usage on
    * `turn.completed`, which never arrives when the turn is aborted (step
    * budget, caller signal); the cumulative count is then read back from the
    * thread's rollout file under this directory.
    */
   codexHome?: string;
+  diagnosticDirectory?: string;
 }): Promise<CodexSessionResult> {
   const sdk = input.sdk ?? (await loadCodexSdk());
   const events: CodexEvent[] = [];
   let finalMessage = "";
   let stopReason: string | undefined;
   let iterationError: unknown;
+  let diagnosticPath: string | undefined;
   let tokenUsage = emptyTokenUsage();
   let usageSource: CodexUsageSource = "none";
   let threadId: string | undefined;
@@ -207,6 +212,15 @@ export async function runCodexSession(input: {
       }
 
       const item = isRecord(event.item) ? event.item : undefined;
+      if (
+        item?.type === "mcp_tool_call" &&
+        input.allowedMcpServers &&
+        !input.allowedMcpServers.includes(String(item.server))
+      ) {
+        const policyError = new Error(`Unexpected MCP server: ${String(item.server)}`);
+        budgetController.abort(policyError);
+        throw policyError;
+      }
       if (
         event.type === "item.completed" &&
         item?.type === "agent_message" &&
@@ -224,11 +238,27 @@ export async function runCodexSession(input: {
           budgetExhausted = true;
           budgetController.abort(new Error(stopReason));
         }
-        if (item.type === "mcp_tool_call") await input.onToolStep?.();
+        if (item.type === "mcp_tool_call") await input.onToolStep?.(item);
       }
     }
   } catch (error) {
     iterationError = error;
+    if (input.diagnosticDirectory) {
+      try {
+        diagnosticPath = await saveCodexDiagnostic(input.diagnosticDirectory, error, events.length);
+        input.logger.warn({
+          category: "codex",
+          level: 0,
+          message: `Codex diagnostic saved: ${diagnosticPath}`,
+        });
+      } catch (diagnosticError) {
+        input.logger.warn({
+          category: "codex",
+          level: 0,
+          message: `Could not save Codex diagnostic: ${sanitizeErrorMessage(stringifyError(diagnosticError))}`,
+        });
+      }
+    }
     input.logger.warn({
       category: "codex",
       message: `Codex stopped before a normal result: ${sanitizeErrorMessage(stringifyError(error))}`,
@@ -267,6 +297,7 @@ export async function runCodexSession(input: {
     usageSource,
     ...(threadId && { threadId }),
     ...(iterationError !== undefined && { iterationError }),
+    ...(diagnosticPath && { diagnosticPath }),
   };
 }
 
@@ -469,7 +500,12 @@ export function isRecord(value: unknown): value is Record<string, unknown> {
 
 export function safeJson(value: unknown): string | undefined {
   try {
-    return JSON.stringify(value);
+    return JSON.stringify(value, (key, item: unknown) => {
+      if (key === "data" && typeof item === "string" && item.length > 256)
+        return `[binary omitted: ${item.length} characters; see trajectory artifact]`;
+      if (typeof item === "string") return clip(sanitizeErrorMessage(item), 32_000);
+      return item;
+    });
   } catch {
     return undefined;
   }