Route existing harness models through native providers

packages/integrations/deepagents/runner/run_eval.py

@@ -408,13 +408,34 @@ def _register_eval_harness_profile(model: str | BaseChatModel) -> None:
     _REGISTERED_PROFILE_KEYS.add(profile_key)
 
 
+def build_eval_model(config: RunnerConfig) -> str | BaseChatModel:
+    """Resolve native provider routing before building the agent."""
+    # xAI's API is OpenAI-compatible; langchain has no xai provider here, so route
+    # grok through ChatOpenAI against api.x.ai (no new dependency, no gateway).
+    if config.model.startswith("xai/") or config.model.startswith("xai:"):
+        from langchain_openai import ChatOpenAI
+
+        api_key = os.environ.get("XAI_API_KEY")
+        if not api_key or not api_key.strip():
+            # ChatOpenAI otherwise falls back to OPENAI_API_KEY, including when
+            # base_url selects another provider. Never send that key to xAI.
+            raise ValueError("XAI_API_KEY is required for xAI models.")
+        model_id = config.model.split("/", 1)[-1].split(":", 1)[-1]
+        return ChatOpenAI(
+            model=model_id,
+            base_url="https://api.x.ai/v1",
+            api_key=api_key,
+        )
+    return config.model
+
+
 def _default_build_agent(
     config: RunnerConfig,
     tools: list[object],
     *,
     model: BaseChatModel | None = None,
 ) -> object:
-    resolved_model: str | BaseChatModel = model or config.model
+    resolved_model: str | BaseChatModel = model or build_eval_model(config)
     _register_eval_harness_profile(resolved_model)
     return create_deep_agent(
         model=resolved_model,

packages/integrations/deepagents/runner/tests/test_model_routing.py

@@ -0,0 +1,55 @@
+from __future__ import annotations
+
+from dataclasses import replace
+from pathlib import Path
+import sys
+
+import langchain_openai
+import pytest
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+from run_eval import RunnerConfig, build_eval_model  # noqa: E402
+
+
+@pytest.mark.parametrize("model", ["xai/grok-fixture", "xai:grok-fixture"])
+def test_xai_uses_native_compatible_endpoint(
+    monkeypatch: pytest.MonkeyPatch, model: str,
+) -> None:
+    configured = {}
+    native_model = object()
+
+    def create_model(**kwargs):
+        configured.update(kwargs)
+        return native_model
+
+    monkeypatch.setattr(langchain_openai, "ChatOpenAI", create_model)
+    monkeypatch.setenv("XAI_API_KEY", "fixture-key")
+    config = RunnerConfig("task", None, model, {}, 10, 5)
+    assert build_eval_model(config) is native_model
+    assert configured == {
+        "model": "grok-fixture",
+        "base_url": "https://api.x.ai/v1",
+        "api_key": "fixture-key",
+    }
+
+
+def test_other_provider_routes_remain_unchanged() -> None:
+    config = RunnerConfig("task", None, "openai:fixture", {}, 10, 5)
+    for model in ["openai:fixture", "anthropic:fixture", "google_genai:fixture"]:
+        assert build_eval_model(replace(config, model=model)) == model
+
+
+@pytest.mark.parametrize("xai_key", [None, "", "   "])
+def test_xai_requires_its_own_key_instead_of_falling_back_to_openai(
+    monkeypatch: pytest.MonkeyPatch, xai_key: str | None,
+) -> None:
+    monkeypatch.setenv("OPENAI_API_KEY", "fixture-openai-key")
+    if xai_key is None:
+        monkeypatch.delenv("XAI_API_KEY", raising=False)
+    else:
+        monkeypatch.setenv("XAI_API_KEY", xai_key)
+    config = RunnerConfig("task", None, "xai/grok-fixture", {}, 10, 5)
+    # Use the installed constructor, which otherwise silently picks OPENAI_API_KEY.
+    # Constructing a model does not make a provider request.
+    with pytest.raises(ValueError, match="XAI_API_KEY is required"):
+        build_eval_model(config)

packages/integrations/mastra-sdk/package.json

@@ -20,6 +20,10 @@
     "typecheck": "tsc --noEmit -p tsconfig.json"
   },
   "dependencies": {
+    "@ai-sdk/anthropic": "catalog:",
+    "@ai-sdk/google": "catalog:",
+    "@ai-sdk/openai": "catalog:",
+    "@ai-sdk/openai-compatible": "^3.0.5",
     "@browserbasehq/stagehand-integrations": "workspace:*",
     "@mastra/core": "catalog:",
     "@mastra/mcp": "catalog:",

packages/integrations/mastra-sdk/src/session.ts

@@ -1,10 +1,17 @@
 import { randomUUID } from "node:crypto";
+import { createAnthropic } from "@ai-sdk/anthropic";
+import { createGoogleGenerativeAI } from "@ai-sdk/google";
+import { createOpenAI } from "@ai-sdk/openai";
+import { createOpenAICompatible } from "@ai-sdk/openai-compatible";
 import {
   HarnessAdapterError,
   sanitizeErrorMessage,
   type HarnessLogger,
 } from "@browserbasehq/stagehand-integrations/harness";
 
+/** A native @ai-sdk model instance (all providers share the LanguageModelV2 shape at one AI-SDK major). */
+type NativeMastraModel = ReturnType<ReturnType<typeof createOpenAI>>;
+
 export type MastraEvent = Record<string, unknown>;
 
 export type MastraStdioServerDefinition = {
@@ -124,6 +131,56 @@ export function normalizeMastraModel(model: string): string {
   return model.includes("/") ? model : `openai/${model}`;
 }
 
+/**
+ * Route first-party models through their native @ai-sdk provider (using the 1p
+ * API keys in the environment) instead of Mastra's default Vercel AI Gateway.
+ * A bare `provider/model` string handed to `createAgent` resolves via the
+ * gateway, which added a failure surface (unparseable error responses ->
+ * sdk_error) and costs more than the native APIs we already hold keys for.
+ * Open models with no native provider (zai/glm, meta/muse, alibaba/qwen, ...)
+ * keep the gateway string. Set MASTRA_FORCE_GATEWAY=1 to disable native routing.
+ */
+export function resolveMastraModel(
+  model: string,
+  env: NodeJS.ProcessEnv = process.env,
+): string | NativeMastraModel {
+  const normalized = normalizeMastraModel(model);
+  if ((env.MASTRA_FORCE_GATEWAY ?? "").trim() === "1") return normalized;
+  // An explicit gateway route is part of the caller's model selection.
+  if (/^(?:vercel|gateway)\//.test(normalized)) return normalized;
+  const bare = normalized;
+  const slash = bare.indexOf("/");
+  if (slash < 0) return normalized;
+  const provider = bare.slice(0, slash);
+  const id = bare.slice(slash + 1);
+  switch (provider) {
+    case "openai": {
+      const apiKey = env.OPENAI_API_KEY;
+      return apiKey ? createOpenAI({ apiKey })(id) : normalized;
+    }
+    case "google": {
+      const apiKey = env.GOOGLE_GENERATIVE_AI_API_KEY ?? env.GEMINI_API_KEY ?? env.GOOGLE_API_KEY;
+      return apiKey ? createGoogleGenerativeAI({ apiKey })(id) : normalized;
+    }
+    case "anthropic": {
+      const apiKey = env.ANTHROPIC_API_KEY;
+      return apiKey ? createAnthropic({ apiKey })(id) : normalized;
+    }
+    case "xai": {
+      // xAI's API is OpenAI-compatible but stricter — the full @ai-sdk/openai
+      // request tripped a 422. Use the openai-compatible provider, which emits a
+      // minimal request (no OpenAI-only params xAI rejects). Same @ai-sdk/provider
+      // spec as mastra's openai@4, so it drops in without a version bump.
+      const apiKey = env.XAI_API_KEY;
+      return apiKey
+        ? createOpenAICompatible({ name: "xai", baseURL: "https://api.x.ai/v1", apiKey })(id)
+        : normalized;
+    }
+    default:
+      return normalized; // open models -> Vercel AI Gateway
+  }
+}
+
 export async function runMastraSession(input: {
   prompt: string;
   model: string;
@@ -198,7 +255,7 @@ export async function runMastraSession(input: {
         name: input.session.agentName ?? "Stagehand Evals Mastra Agent",
         instructions:
           input.session.instructions ?? "Use the available browser/web tools to complete the task.",
-        model: normalizeMastraModel(input.model),
+        model: resolveMastraModel(input.model),
         tools: { ...mcpTools, ...input.session.tools },
       });
       const stream = await agent.stream(input.prompt, {

packages/integrations/mastra-sdk/tests/model-routing.test.ts

@@ -0,0 +1,28 @@
+import { describe, expect, it } from "vitest";
+import { resolveMastraModel } from "../src/session.js";
+
+describe("Mastra model routing", () => {
+  it.each([
+    ["openai/example", "OPENAI_API_KEY"],
+    ["anthropic/example", "ANTHROPIC_API_KEY"],
+    ["google/example", "GOOGLE_API_KEY"],
+    ["xai/example", "XAI_API_KEY"],
+  ])("uses the native provider for %s when its credential is present", (model, key) => {
+    expect(resolveMastraModel(model, { [key]: "fixture-key" })).toMatchObject({
+      modelId: "example",
+    });
+    expect(resolveMastraModel(model, {})).toBe(model);
+  });
+
+  it("respects an explicit gateway route and the force-gateway override", () => {
+    expect(resolveMastraModel("gateway/openai/example", { OPENAI_API_KEY: "fixture-key" })).toBe(
+      "gateway/openai/example",
+    );
+    expect(
+      resolveMastraModel("openai/example", {
+        OPENAI_API_KEY: "fixture-key",
+        MASTRA_FORCE_GATEWAY: "1",
+      }),
+    ).toBe("openai/example");
+  });
+});