feat: vertex ai openai adapter (#6736)

* feat: vertex ai openai adapter

* fix: vertex ai - missing mistral key

* feat: support tools for vertexAI gemini and anthropic

* feat: vertex openai adapter working for gemini

* fix: show errors in vertex ai

* fix: filter out model from vertexai anthropic body

* feat: vertexai toolSupport update

* chore: remove force openai adapter usage for vertex

* fix: prettier

Author: RomneyDa

core/llm/llms/Anthropic.ts

@@ -172,52 +172,10 @@ class Anthropic extends BaseLLM {
     }
   }
 
-  protected async *_streamChat(
-    messages: ChatMessage[],
-    signal: AbortSignal,
-    options: CompletionOptions,
+  async *handleResponse(
+    response: any,
+    stream: boolean | undefined,
   ): AsyncGenerator<ChatMessage> {
-    if (!this.apiKey || this.apiKey === "") {
-      throw new Error(
-        "Request not sent. You have an Anthropic model configured in your config.json, but the API key is not set.",
-      );
-    }
-
-    const systemMessage = stripImages(
-      messages.filter((m) => m.role === "system")[0]?.content ?? "",
-    );
-    const shouldCacheSystemMessage = !!(
-      this.cacheBehavior?.cacheSystemMessage && systemMessage
-    );
-
-    const msgs = this.convertMessages(messages);
-    const response = await this.fetch(new URL("messages", this.apiBase), {
-      method: "POST",
-      headers: {
-        "Content-Type": "application/json",
-        Accept: "application/json",
-        "anthropic-version": "2023-06-01",
-        "x-api-key": this.apiKey as string,
-        ...(shouldCacheSystemMessage || this.cacheBehavior?.cacheConversation
-          ? { "anthropic-beta": "prompt-caching-2024-07-31" }
-          : {}),
-      },
-      body: JSON.stringify({
-        ...this.convertArgs(options),
-        messages: msgs,
-        system: shouldCacheSystemMessage
-          ? [
-              {
-                type: "text",
-                text: systemMessage,
-                cache_control: { type: "ephemeral" },
-              },
-            ]
-          : systemMessage,
-      }),
-      signal,
-    });
-
     if (response.status === 499) {
       return; // Aborted by user
     }
@@ -237,7 +195,7 @@ class Anthropic extends BaseLLM {
       );
     }
 
-    if (options.stream === false) {
+    if (stream === false) {
       const data = await response.json();
       const cost = data.usage
         ? {
@@ -348,6 +306,55 @@ class Anthropic extends BaseLLM {
       usage,
     };
   }
+
+  protected async *_streamChat(
+    messages: ChatMessage[],
+    signal: AbortSignal,
+    options: CompletionOptions,
+  ): AsyncGenerator<ChatMessage> {
+    if (!this.apiKey || this.apiKey === "") {
+      throw new Error(
+        "Request not sent. You have an Anthropic model configured in your config.json, but the API key is not set.",
+      );
+    }
+
+    const systemMessage = stripImages(
+      messages.filter((m) => m.role === "system")[0]?.content ?? "",
+    );
+    const shouldCacheSystemMessage = !!(
+      this.cacheBehavior?.cacheSystemMessage && systemMessage
+    );
+
+    const msgs = this.convertMessages(messages);
+    const response = await this.fetch(new URL("messages", this.apiBase), {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        Accept: "application/json",
+        "anthropic-version": "2023-06-01",
+        "x-api-key": this.apiKey as string,
+        ...(shouldCacheSystemMessage || this.cacheBehavior?.cacheConversation
+          ? { "anthropic-beta": "prompt-caching-2024-07-31" }
+          : {}),
+      },
+      body: JSON.stringify({
+        ...this.convertArgs(options),
+        messages: msgs,
+        system: shouldCacheSystemMessage
+          ? [
+              {
+                type: "text",
+                text: systemMessage,
+                cache_control: { type: "ephemeral" },
+              },
+            ]
+          : systemMessage,
+      }),
+      signal,
+    });
+
+    yield* this.handleResponse(response, options.stream);
+  }
 }
 
 export default Anthropic;

core/llm/llms/Gemini.ts

@@ -200,6 +200,7 @@ class Gemini extends BaseLLM {
     messages: ChatMessage[],
     options: CompletionOptions,
     isV1API: boolean,
+    includeToolIds: boolean,
   ): GeminiChatRequestBody {
     const toolCallIdToNameMap = new Map<string, string>();
     messages.forEach((msg) => {
@@ -231,7 +232,7 @@ class Gemini extends BaseLLM {
               parts: [
                 {
                   functionResponse: {
-                    id: msg.toolCallId,
+                    id: includeToolIds ? msg.toolCallId : undefined,
                     name: functionName || "unknown",
                     response: {
                       output: msg.content, // "output" key is opinionated - not all functions will output objects
@@ -406,7 +407,7 @@ class Gemini extends BaseLLM {
     const isV1API = !!this.apiBase?.includes("/v1/");
 
     // Convert chat messages to contents
-    const body = this.prepareBody(messages, options, isV1API);
+    const body = this.prepareBody(messages, options, isV1API, true);
 
     const response = await this.fetch(apiURL, {
       method: "POST",

core/llm/llms/Mistral.ts

@@ -67,7 +67,7 @@ class Mistral extends OpenAI {
     return Mistral.modelConversion[model] ?? model;
   }
 
-  protected _convertArgs(options: any, messages: ChatMessage[]) {
+  _convertArgs(options: any, messages: ChatMessage[]) {
     const finalOptions = super._convertArgs(options, messages);
 
     const lastMessage = finalOptions.messages[finalOptions.messages.length - 1];

core/llm/llms/VertexAI.ts

@@ -254,24 +254,7 @@ class VertexAI extends BaseLLM {
       signal,
     });
 
-    if (response.status === 499) {
-      return; // Aborted by user
-    }
-
-    if (options.stream === false) {
-      const data = await response.json();
-      yield { role: "assistant", content: data.content[0].text };
-      return;
-    }
-
-    for await (const value of streamSse(response)) {
-      if (value.type === "message_start") {
-        console.log(value);
-      }
-      if (value.delta?.text) {
-        yield { role: "assistant", content: value.delta.text };
-      }
-    }
+    yield* this.anthropicInstance.handleResponse(response, options.stream);
   }
 
   // Gemini
@@ -285,17 +268,19 @@ class VertexAI extends BaseLLM {
       this.apiBase,
     );
 
-    const body = this.geminiInstance.prepareBody(messages, options, false);
+    // For some reason gemini through vertex does not support ids in functionResponses yet
+    const body = this.geminiInstance.prepareBody(
+      messages,
+      options,
+      false,
+      false,
+    );
     const response = await this.fetch(apiURL, {
       method: "POST",
       body: JSON.stringify(body),
       signal,
     });
-    for await (const message of this.geminiInstance.processGeminiResponse(
-      streamResponse(response),
-    )) {
-      yield message;
-    }
+    yield* this.geminiInstance.processGeminiResponse(streamResponse(response));
   }
 
   private async *streamChatBison(

core/llm/toolSupport.ts

@@ -83,11 +83,12 @@ export const PROVIDER_TOOL_SUPPORT: Record<string, (model: string) => boolean> =
       return model.toLowerCase().includes("gemini");
     },
     vertexai: (model) => {
+      const lowerCaseModel = model.toLowerCase();
       // All gemini models except flash 2.0 lite support function calling
-      return (
-        model.toLowerCase().includes("gemini") &&
-        !model.toLowerCase().includes("lite")
-      );
+      if (lowerCaseModel.includes("lite")) {
+        return false;
+      }
+      return ["claude", "gemini"].some((val) => lowerCaseModel.includes(val));
     },
     bedrock: (model) => {
       if (