Add Gemini native computer use with model-visible evidence

.github/workflows/ci.yml

@@ -261,6 +261,7 @@ jobs:
             packages/integrations/core/dist/**
             packages/integrations/claude-agent-sdk/dist/**
             packages/integrations/claude-cua-sdk/dist/**
+            packages/integrations/gemini-cua-sdk/dist/**
             packages/integrations/codex-sdk/dist/**
             packages/integrations/mastra-sdk/dist/**
             packages/integrations/pi-sdk/dist/**

packages/evals/README.md

@@ -1,6 +1,6 @@
 # Stagehand Evals
 
-Agent benchmarks for Stagehand — `act`, `extract`, `observe`, `agent`, plus dataset-backed suites (WebVoyager, OnlineMind2Web, WebTailBench, Odysseys).
+Agent benchmarks for Stagehand — `act`, `extract`, `observe`, `agent`, plus dataset-backed suites (WebVoyager, OnlineMind2Web, WebTailBench, Odysseys, HardBench).
 
 Driven by an interactive TUI (`evals`) or single-shot CLI (`evals run …`). Tasks are auto-discovered from `tasks/bench/<category>/` — no registration step.
 
@@ -84,6 +84,12 @@ A live run paints an in-place progress table, then prints a final summary with a
 
 ![Live bench run](./assets/readme/run.gif)
 
+## Shared harness behavior
+
+See [the harness contract](docs/harness-contract.md) for tool surfaces, prompt policy,
+budget units, session diagnostics, verification, and usage accounting.
+See [HardBench](datasets/hardbenchmark/README.md) for corpus selection and rubric v1.2.
+
 ## Adding a bench task
 
 ```bash

packages/evals/core/contracts/tool.ts

@@ -11,6 +11,7 @@ export type ToolSurface =
   | "cdp_code"
   | "playwright_mcp"
   | "chrome_devtools_mcp"
+  | "google_computer_use"
   | "anthropic_browser_toolset"
   | "stagehand_facade"
   | "stagehand_facade_legacy"

packages/evals/core/tools/registry.ts

@@ -9,11 +9,13 @@ import {
   StagehandFacadeTool,
   StagehandFacadeLegacyTool,
   AnthropicBrowserToolsetTool,
+  GoogleComputerUseTool,
 } from "./stagehand_facade.js";
 import { UnderstudyCodeTool } from "./understudy_code.js";
 
 /** Surfaces that exist only as an agent MCP mount; they have no runner-driven CoreSession (activePage() throws). */
 export const AGENT_MOUNT_ONLY_TOOL_SURFACES: ReadonlySet<ToolSurface> = new Set<ToolSurface>([
+  "google_computer_use",
   "anthropic_browser_toolset",
   "stagehand_facade",
   "stagehand_facade_legacy",
@@ -33,6 +35,7 @@ export function listCoreTools(): ToolSurface[] {
     "chrome_devtools_mcp",
     // Listed here as part of the full enumeration, but agent-mount-only:
     // core-tier selection must use listCoreRunnableTools, which filters it.
+    "google_computer_use",
     "anthropic_browser_toolset",
     "stagehand_facade",
     "stagehand_facade_legacy",
@@ -59,6 +62,8 @@ export function getCoreTool(toolSurface: ToolSurface): CoreTool {
       return new PlaywrightMcpTool();
     case "chrome_devtools_mcp":
       return new ChromeDevtoolsMcpTool();
+    case "google_computer_use":
+      return new GoogleComputerUseTool();
     case "anthropic_browser_toolset":
       return new AnthropicBrowserToolsetTool();
     case "stagehand_facade":

packages/evals/core/tools/stagehand_facade.ts

@@ -326,3 +326,8 @@ export class StagehandFacadeLegacyTool extends StagehandFacadeTool {
 export class AnthropicBrowserToolsetTool extends StagehandFacadeTool {
   override readonly id: ToolSurface = "anthropic_browser_toolset";
 }
+
+/** Native Gemini actions execute on the same facade-owned browser. */
+export class GoogleComputerUseTool extends StagehandFacadeTool {
+  override readonly id: ToolSurface = "google_computer_use";
+}

packages/evals/docs/harness-contract.md

@@ -0,0 +1,60 @@
+# Shared eval harness contract
+
+External harnesses use the runner-owned tool mount. The default is `stagehand_facade`
+where supported; `stagehand_facade_legacy` remains explicitly selectable. Provider
+adapters translate their protocols to that shared runtime. They do not start a
+second browser or implement a separate Playwright facade.
+
+## Prompt and execution configuration
+
+The eval runner owns the no-clarification policy in `evalSystemPrompt.ts`. It is
+injected once through an additive system/developer channel when available;
+otherwise it is included once in the task prompt. Harness stock prompts remain
+in place. A caller-supplied Codex client and remote Eve use the task-prefix path.
+The Cursor SDK's replacement system-prompt option is intentionally unused.
+
+Budget precedence is the harness-specific environment variable, then
+`AGENT_EVAL_MAX_STEPS`, then the dataset default (HardBench: 100), then the
+historical harness default. Only positive safe integers are accepted.
+`harnessConfiguration` records the effective budget, unit, policy channel/version,
+and requested reasoning settings where supported. Equal budget numbers do not
+mean equal work:
+
+| Harness                                 | Counted unit          |
+| --------------------------------------- | --------------------- |
+| Codex, Cursor, DeepAgents               | Tool calls            |
+| Eve                                     | Successful tool calls |
+| Mastra                                  | Model steps           |
+| fx                                      | Agent steps           |
+| Claude Code, Pi, Claude CUA, Gemini CUA | Turns                 |
+
+These are execution limits, not comparable measures of model efficiency.
+
+## Session and result records
+
+The mounted surface supplies session identity and evidence. Session ownership,
+first terminal browser loss, and cleanup live in the shared runtime. Capture
+recovery permits two consecutive capture deadlines; the third ends the session.
+A successful capture resets the counter. Timed-out work cannot replace newer
+snapshot IDs, and failed actions are not replayed. CDP heartbeat and sanitized
+`CDP_DROP` diagnostics are available; automatic reconnect is not implemented.
+
+`harnessStatus` and `terminationReason` describe execution. The verifier determines
+completion from evidence, including work completed before a late disconnect.
+When verification fails, `_success` is false, `verifierError` records the failure,
+and the agent's report is preserved separately. Captured trajectories and raw
+judge uncertainty are retained when persistence is enabled. See
+[verification gates](verifier-gates.md) for scoring and audit fields.
+
+## Usage and cost
+
+Raw token usage is normalized according to the SDK's cache conventions.
+Observed zero is distinct from missing telemetry; missing usage and unknown
+subscription bills do not become zero-dollar runs. Reported bills take precedence.
+Direct-provider estimates use the dated checked-in price map and include
+`cost_source: computed` and `cost_pricing` (date, matched model, source).
+These are list-price estimates, not invoice reconciliation or current-price claims.
+
+Cursor runs use the SDK through `--harness cursor` and record implementation/version
+provenance. Historical CLI `cursor` and `cursor_sdk` records remain readable;
+missing provenance is not retroactively interpreted as an SDK run.

packages/evals/framework/benchHarness.ts

@@ -1,3 +1,5 @@
+import { runGeminiCuaAgent, GEMINI_CUA_DEFAULT_MODELS } from "./geminiCuaRunner.js";
+import { GEMINI_CUA_TOOL_SURFACES, prepareGeminiCuaToolAdapter } from "./geminiCuaToolAdapter.js";
 import { runClaudeCuaAgent, CLAUDE_CUA_DEFAULT_MODELS } from "./claudeCuaRunner.js";
 import { CLAUDE_CUA_TOOL_SURFACES, prepareClaudeCuaToolAdapter } from "./claudeCuaToolAdapter.js";
 import { V3, normalizeRubric, type AvailableModel, type TaskSpec } from "stagehand-v3";
@@ -389,6 +391,14 @@ export const claudeCuaHarness = defineExternalHarness({
   runAgent: runClaudeCuaAgent,
 });
 
+export const geminiCuaHarness = defineExternalHarness({
+  harness: "gemini_cua",
+  supportedToolSurfaces: GEMINI_CUA_TOOL_SURFACES,
+  defaultModels: GEMINI_CUA_DEFAULT_MODELS,
+  prepareToolAdapter: prepareGeminiCuaToolAdapter,
+  runAgent: runGeminiCuaAgent,
+});
+
 const harnessRegistry = new Map<Harness, BenchHarness>([
   ["stagehand", stagehandHarness],
   ["claude_code", claudeCodeHarness],
@@ -400,6 +410,7 @@ const harnessRegistry = new Map<Harness, BenchHarness>([
   ["fx", fxHarness],
   ["cursor", cursorHarness],
   ["claude_cua", claudeCuaHarness],
+  ["gemini_cua", geminiCuaHarness],
 ]);
 
 export function registerBenchHarness(harness: BenchHarness): () => void {

packages/evals/framework/geminiCuaRunner.ts

@@ -0,0 +1,131 @@
+import {
+  buildGeminiCuaTranscript,
+  runGeminiCuaSession,
+  normalizeGeminiCuaModel,
+  type GeminiGenerateClient,
+} from "@browserbasehq/stagehand-integrations-gemini-cua-sdk";
+import type { AvailableModel } from "stagehand-v3";
+import { EvalsError } from "../errors.js";
+import type { EvalLogger } from "../logger.js";
+import type { PreparedGeminiCuaToolAdapter } from "./geminiCuaToolAdapter.js";
+import type { ExternalHarnessTaskPlan } from "./externalHarnessPlan.js";
+import {
+  buildExternalHarnessPrompt,
+  metricValue,
+  runExternalHarnessTask,
+} from "./harnesses/externalRunner.js";
+import { resolveStepBudget } from "./stepBudget.js";
+import type { TaskResult } from "./types.js";
+import type { ExternalHarnessVerifierConfig } from "./verifierAdapter.js";
+import { geminiCuaAdapter } from "./harnesses/geminiCuaAdapter.js";
+export const GEMINI_CUA_DEFAULT_MODELS: AvailableModel[] = [
+  "google/gemini-3.8-flash" as AvailableModel,
+];
+export const GEMINI_CUA_SYSTEM_PROMPT =
+  "You are being evaluated on a browser task. Complete it with computer use only and finish by printing the requested EVAL_RESULT line.";
+export interface GeminiCuaRunnerInput {
+  plan: ExternalHarnessTaskPlan;
+  model: AvailableModel;
+  logger: EvalLogger;
+  toolAdapter: PreparedGeminiCuaToolAdapter;
+  signal?: AbortSignal;
+  verifier?: ExternalHarnessVerifierConfig;
+  client?: GeminiGenerateClient;
+}
+export function buildGeminiCuaPrompt(
+  plan: ExternalHarnessTaskPlan,
+  toolInstructions?: string,
+): string {
+  return buildExternalHarnessPrompt({ plan, toolInstructions, resultContract: "marker" });
+}
+export function assertGeminiCuaModel(model: string): void {
+  try {
+    normalizeGeminiCuaModel(model);
+  } catch {
+    throw new EvalsError("gemini_cua requires a non-empty model identifier.");
+  }
+}
+export async function runGeminiCuaAgent(input: GeminiCuaRunnerInput): Promise<TaskResult> {
+  assertGeminiCuaModel(input.model);
+  const maxTurns = resolveStepBudget({
+    harnessEnvKey: "EVAL_GEMINI_CUA_MAX_TURNS",
+    dataset: input.plan.dataset,
+    harnessDefault: 50,
+  });
+  return runExternalHarnessTask({
+    harness: "gemini_cua",
+    plan: input.plan,
+    model: input.model,
+    logger: input.logger,
+    toolAdapter: input.toolAdapter,
+    verifier: input.verifier,
+    resultContract: "marker",
+    fallbackErrorMessage: "gemini_cua did not report success",
+    stepBudget: maxTurns,
+    stepBudgetUnit: "turns",
+    systemPromptMode: "native",
+    runSession: async (prompt, systemPrompt) => {
+      const result = await runGeminiCuaSession({
+        prompt,
+        model: input.model,
+        logger: input.logger,
+        maxTurns,
+        tools: input.toolAdapter.executor,
+        facade: input.toolAdapter.facade,
+        systemPrompt: `${systemPrompt}\n\n${GEMINI_CUA_SYSTEM_PROMPT}`,
+        signal: input.signal,
+        client: input.client,
+        browserSessionLoss: input.toolAdapter.browserSessionLoss,
+      });
+      return {
+        raw: result,
+        resultText: result.finalMessage,
+        transcriptText: buildGeminiCuaTranscript(result.events),
+        iterationError: result.iterationError,
+        status: result.status,
+        stopReason:
+          result.status === "sdk_error"
+            ? result.stopReason || stringifyError(result.iterationError) || undefined
+            : result.stopReason,
+        usage: {
+          reported: result.usageReported,
+          inputTokens: result.tokenUsage.input,
+          outputTokens: result.tokenUsage.output,
+          cachedInputTokens: result.tokenUsage.cached_input,
+          reasoningOutputTokens: result.tokenUsage.reasoning,
+          totalTokens: result.tokenUsage.total,
+        },
+        metrics: {
+          gemini_cua_turns: metricValue(result.turns),
+          gemini_cua_tool_calls: metricValue(result.toolCalls),
+        },
+      };
+    },
+    toTrajectory: ({ raw, parsed, finalObservation, status }, taskSpec) =>
+      geminiCuaAdapter.fromHarnessResult(
+        {
+          events: raw.events,
+          finalAnswer: parsed.finalAnswer ?? raw.finalMessage,
+          status,
+          ...(finalObservation && { finalObservation }),
+          usage: {
+            input_tokens: raw.tokenUsage.input,
+            output_tokens: raw.tokenUsage.output,
+            cached_input_tokens: raw.tokenUsage.cached_input,
+          },
+        },
+        taskSpec,
+      ),
+  });
+}
+
+function stringifyError(value: unknown): string {
+  if (!value) return "";
+  if (value instanceof Error) return value.message;
+  if (typeof value === "string") return value;
+  try {
+    return JSON.stringify(value) ?? String(value);
+  } catch {
+    return String(value);
+  }
+}

packages/evals/framework/geminiCuaToolAdapter.ts

@@ -0,0 +1,87 @@
+import {
+  GeminiCuaExecutor,
+  type CuaFacadeTools,
+  type GeminiToolExecutor,
+} from "@browserbasehq/stagehand-integrations-gemini-cua-sdk";
+import type { ProbeEvidence } from "stagehand-v3";
+import type { BrowserSessionLoss, StartupProfile, ToolSurface } from "../core/contracts/tool.js";
+import { bridgeCuaFacadeTools, cuaCleanup } from "./cuaToolAdapter.js";
+import { EvalsError } from "../errors.js";
+import type { EvalLogger } from "../logger.js";
+import type { BrowserSessionInfo } from "./browserSession.js";
+import type { ExternalHarnessTaskPlan } from "./externalHarnessPlan.js";
+import { startAgentToolRuntime } from "./agentToolRuntime.js";
+import { resolveStartupProfile, resolveToolSurface } from "./harnesses/toolSurfaceResolution.js";
+
+export const GEMINI_CUA_TOOL_INSTRUCTIONS =
+  "Browser tool surface: Gemini Computer Use.\nUse the computer-use actions supplied by the API to control the existing browser. Action coordinates use the normalized 0–999 range on a 1288×711 viewport. Each action response includes the current URL and a fresh screenshot when available. Use the supplied action schemas for their exact arguments.";
+
+export const GEMINI_CUA_TOOL_SURFACES: ToolSurface[] = ["google_computer_use"];
+export interface PreparedGeminiCuaToolAdapter {
+  toolSurface: ToolSurface;
+  startupProfile: StartupProfile;
+  promptInstructions: string;
+  browserSession: BrowserSessionInfo;
+  executor: GeminiToolExecutor;
+  facade: CuaFacadeTools;
+  captureEvidence?: () => Promise<ProbeEvidence>;
+  browserSessionLoss?: () => BrowserSessionLoss | undefined;
+  observedToolMatcher: (name: string) => boolean;
+  cleanup: () => Promise<void>;
+}
+
+export async function prepareGeminiCuaToolAdapter(input: {
+  toolSurface?: ToolSurface;
+  startupProfile?: StartupProfile;
+  environment: "LOCAL" | "BROWSERBASE";
+  plan: ExternalHarnessTaskPlan;
+  logger: EvalLogger;
+}): Promise<PreparedGeminiCuaToolAdapter> {
+  const toolSurface = resolveToolSurface(
+    { harness: "gemini_cua", supportedToolSurfaces: GEMINI_CUA_TOOL_SURFACES },
+    input.toolSurface,
+  );
+  if (!toolSurface) throw new EvalsError("gemini_cua harness requires a tool surface.");
+  const startupProfile = resolveStartupProfile(
+    toolSurface,
+    input.environment,
+    input.startupProfile,
+  );
+  const runtime = await startAgentToolRuntime({
+    toolSurface,
+    startupProfile,
+    environment: input.environment,
+    logger: input.logger,
+  });
+  const cleanup = cuaCleanup(runtime.cleanup);
+  try {
+    const callTool = runtime.running.callTool;
+    const mount = runtime.running.agentMount;
+    if (!callTool || !mount)
+      throw new EvalsError(`Tool surface "${toolSurface}" does not expose runner-side tool calls.`);
+    const facade = bridgeCuaFacadeTools(callTool, undefined, runtime.running.browserSessionLoss);
+    const executor = new GeminiCuaExecutor(
+      facade,
+      input.logger,
+      undefined,
+      runtime.running.browserSessionLoss,
+    );
+    return {
+      toolSurface,
+      startupProfile,
+      promptInstructions: GEMINI_CUA_TOOL_INSTRUCTIONS,
+      browserSession: runtime.browserSession,
+      executor,
+      facade,
+      ...(runtime.running.captureEvidence && { captureEvidence: runtime.running.captureEvidence }),
+      ...(runtime.running.browserSessionLoss && {
+        browserSessionLoss: runtime.running.browserSessionLoss,
+      }),
+      observedToolMatcher: () => true,
+      cleanup,
+    };
+  } catch (error) {
+    await cleanup();
+    throw error;
+  }
+}