Merge pull request #235 from langchain-ai/brace/file-support

feat: Support uploading files

package.json

@@ -35,12 +35,12 @@
     "@codemirror/lang-xml": "^6.1.0",
     "@langchain/anthropic": "^0.3.12",
     "@langchain/community": "^0.3.26",
-    "@langchain/core": "^0.3.33",
-    "@langchain/google-genai": "^0.1.6",
+    "@langchain/core": "^0.3.36",
+    "@langchain/google-genai": "^0.1.7",
     "@langchain/langgraph": "^0.2.41",
     "@langchain/langgraph-sdk": "^0.0.36",
     "@langchain/ollama": "^0.1.4",
-    "@langchain/openai": "^0.3.17",
+    "@langchain/openai": "^0.4.2",
     "@nextjournal/lang-clojure": "^1.0.0",
     "@radix-ui/react-avatar": "^1.1.0",
     "@radix-ui/react-checkbox": "^1.1.2",
@@ -77,6 +77,7 @@
     "lodash": "^4.17.21",
     "lucide-react": "^0.441.0",
     "next": "14.2.10",
+    "pdf-parse": "^1.1.1",
     "react": "^18",
     "react-colorful": "^5.6.1",
     "react-dom": "^18",
@@ -99,6 +100,7 @@
     "@types/js-cookie": "^3.0.6",
     "@types/lodash": "^4.17.12",
     "@types/node": "^20",
+    "@types/pdf-parse": "^1.1.4",
     "@types/react": "^18",
     "@types/react-dom": "^18",
     "@types/uuid": "^10.0.0",

src/agent/open-canvas/nodes/customAction.ts

@@ -49,7 +49,7 @@ export const customAction = async (
     throw new Error("`assistant_id` not found in configurable");
   }
   if (!userId) {
-    throw new Error("`supabase_user_id` not found in configurable");
+    throw new Error("`user.id` not found in configurable");
   }
   const customActionsNamespace = ["custom_actions", userId];
   const actionsKey = "actions";

src/agent/open-canvas/nodes/generate-artifact/index.ts

@@ -1,7 +1,9 @@
 import {
+  createContextDocumentMessages,
   getFormattedReflections,
   getModelConfig,
   getModelFromConfig,
+  isUsingO1MiniModel,
   optionallyGetSystemPromptFromConfig,
 } from "@/agent/utils";
 import { ArtifactV3 } from "@/types";
@@ -20,9 +22,12 @@ export const generateArtifact = async (
   state: typeof OpenCanvasGraphAnnotation.State,
   config: LangGraphRunnableConfig
 ): Promise<OpenCanvasGraphReturnType> => {
-  const { modelName, modelProvider } = getModelConfig(config);
+  const { modelName, modelProvider } = getModelConfig(config, {
+    isToolCalling: true,
+  });
   const smallModel = await getModelFromConfig(config, {
     temperature: 0.5,
+    isToolCalling: true,
   });
 
   const modelWithArtifactTool = smallModel.bindTools(
@@ -47,8 +52,14 @@ export const generateArtifact = async (
     ? `${userSystemPrompt}\n${formattedNewArtifactPrompt}`
     : formattedNewArtifactPrompt;
 
+  const contextDocumentMessages = await createContextDocumentMessages(config);
+  const isO1MiniModel = isUsingO1MiniModel(config);
   const response = await modelWithArtifactTool.invoke(
-    [{ role: "system", content: fullSystemPrompt }, ...state.messages],
+    [
+      { role: isO1MiniModel ? "user" : "system", content: fullSystemPrompt },
+      ...contextDocumentMessages,
+      ...state.messages,
+    ],
     { runName: "generate_artifact" }
   );
 

src/agent/open-canvas/nodes/generatePath.ts

@@ -2,6 +2,7 @@ import { LangGraphRunnableConfig } from "@langchain/langgraph";
 import { z } from "zod";
 import { getArtifactContent } from "../../../contexts/utils";
 import {
+  createContextDocumentMessages,
   formatArtifactContentWithTemplate,
   getModelFromConfig,
 } from "../../utils";
@@ -94,6 +95,7 @@ export const generatePath = async (
 
   const model = await getModelFromConfig(config, {
     temperature: 0,
+    isToolCalling: true,
   });
   const modelWithTool = model.withStructuredOutput(
     z.object({
@@ -106,7 +108,9 @@ export const generatePath = async (
     }
   );
 
+  const contextDocumentMessages = await createContextDocumentMessages(config);
   const result = await modelWithTool.invoke([
+    ...contextDocumentMessages,
     {
       role: "user",
       content: formattedPrompt,

src/agent/open-canvas/nodes/replyToGeneralInput.ts

@@ -2,10 +2,12 @@ import { LangGraphRunnableConfig } from "@langchain/langgraph";
 import { getArtifactContent } from "../../../contexts/utils";
 import { Reflections } from "../../../types";
 import {
+  createContextDocumentMessages,
   ensureStoreInConfig,
   formatArtifactContentWithTemplate,
   formatReflections,
   getModelFromConfig,
+  isUsingO1MiniModel,
 } from "../../utils";
 import { CURRENT_ARTIFACT_PROMPT, NO_ARTIFACT_PROMPT } from "../prompts";
 import { OpenCanvasGraphAnnotation, OpenCanvasGraphReturnType } from "../state";
@@ -58,8 +60,11 @@ You also have the following reflections on style guidelines and general memories
         : NO_ARTIFACT_PROMPT
     );
 
+  const contextDocumentMessages = await createContextDocumentMessages(config);
+  const isO1MiniModel = isUsingO1MiniModel(config);
   const response = await smallModel.invoke([
-    { role: "system", content: formattedPrompt },
+    { role: isO1MiniModel ? "user" : "system", content: formattedPrompt },
+    ...contextDocumentMessages,
     ...state.messages,
   ]);
 

src/agent/open-canvas/nodes/rewrite-artifact/index.ts

@@ -6,8 +6,10 @@ import { LangGraphRunnableConfig } from "@langchain/langgraph";
 import { optionallyUpdateArtifactMeta } from "./update-meta";
 import { buildPrompt, createNewArtifactContent, validateState } from "./utils";
 import {
+  createContextDocumentMessages,
   getFormattedReflections,
   getModelFromConfig,
+  isUsingO1MiniModel,
   optionallyGetSystemPromptFromConfig,
 } from "@/agent/utils";
 import { isArtifactMarkdownContent } from "@/lib/artifact_content_types";
@@ -45,8 +47,11 @@ export const rewriteArtifact = async (
     ? `${userSystemPrompt}\n${formattedPrompt}`
     : formattedPrompt;
 
+  const contextDocumentMessages = await createContextDocumentMessages(config);
+  const isO1MiniModel = isUsingO1MiniModel(config);
   const newArtifactResponse = await smallModelWithConfig.invoke([
-    { role: "system", content: fullSystemPrompt },
+    { role: isO1MiniModel ? "user" : "system", content: fullSystemPrompt },
+    ...contextDocumentMessages,
     recentHumanMessage,
   ]);
 

src/agent/open-canvas/nodes/rewrite-artifact/update-meta.ts

@@ -4,6 +4,7 @@ import {
   formatArtifactContent,
   getModelConfig,
   getModelFromConfig,
+  isUsingO1MiniModel,
 } from "@/agent/utils";
 import { getArtifactContent } from "@/contexts/utils";
 import { GET_TITLE_TYPE_REWRITE_ARTIFACT } from "../../prompts";
@@ -15,8 +16,14 @@ export async function optionallyUpdateArtifactMeta(
   state: typeof OpenCanvasGraphAnnotation.State,
   config: LangGraphRunnableConfig
 ): Promise<ToolCall | undefined> {
-  const { modelProvider } = getModelConfig(config);
-  const toolCallingModel = (await getModelFromConfig(config))
+  const { modelProvider } = getModelConfig(config, {
+    isToolCalling: true,
+  });
+  const toolCallingModel = (
+    await getModelFromConfig(config, {
+      isToolCalling: true,
+    })
+  )
     .bindTools(
       [
         {
@@ -56,8 +63,12 @@ export async function optionallyUpdateArtifactMeta(
     throw new Error("No recent human message found");
   }
 
+  const isO1MiniModel = isUsingO1MiniModel(config);
   const optionallyUpdateArtifactResponse = await toolCallingModel.invoke([
-    { role: "system", content: optionallyUpdateArtifactMetaPrompt },
+    {
+      role: isO1MiniModel ? "user" : "system",
+      content: optionallyUpdateArtifactMetaPrompt,
+    },
     recentHumanMessage,
   ]);
 

src/agent/open-canvas/nodes/updateArtifact.ts

@@ -3,10 +3,12 @@ import { getArtifactContent } from "../../../contexts/utils";
 import { isArtifactCodeContent } from "../../../lib/artifact_content_types";
 import { ArtifactCodeV3, ArtifactV3, Reflections } from "../../../types";
 import {
+  createContextDocumentMessages,
   ensureStoreInConfig,
   formatReflections,
   getModelConfig,
   getModelFromConfig,
+  isUsingO1MiniModel,
 } from "../../utils";
 import { UPDATE_HIGHLIGHTED_ARTIFACT_PROMPT } from "../prompts";
 import { OpenCanvasGraphAnnotation, OpenCanvasGraphReturnType } from "../state";
@@ -18,15 +20,15 @@ export const updateArtifact = async (
   state: typeof OpenCanvasGraphAnnotation.State,
   config: LangGraphRunnableConfig
 ): Promise<OpenCanvasGraphReturnType> => {
-  const { modelProvider } = getModelConfig(config);
+  const { modelProvider, modelName } = getModelConfig(config);
   let smallModel: Awaited<ReturnType<typeof getModelFromConfig>>;
-  if (modelProvider.includes("openai")) {
-    // Custom model is OpenAI/Azure OpenAI
+  if (modelProvider.includes("openai") || modelName.includes("3-5-sonnet")) {
+    // Custom model is intelligent enough for updating artifacts
     smallModel = await getModelFromConfig(config, {
       temperature: 0,
     });
   } else {
-    // Custom model is not set to OpenAI/Azure OpenAI. Use GPT-4o
+    // Custom model is not intelligent enough for updating artifacts
     smallModel = await getModelFromConfig(
       {
         ...config,
@@ -102,8 +104,12 @@ export const updateArtifact = async (
   if (!recentHumanMessage) {
     throw new Error("No recent human message found");
   }
+
+  const contextDocumentMessages = await createContextDocumentMessages(config);
+  const isO1MiniModel = isUsingO1MiniModel(config);
   const updatedArtifact = await smallModel.invoke([
-    { role: "system", content: formattedPrompt },
+    { role: isO1MiniModel ? "user" : "system", content: formattedPrompt },
+    ...contextDocumentMessages,
     recentHumanMessage,
   ]);
 

src/agent/open-canvas/nodes/updateHighlightedText.ts

@@ -1,4 +1,9 @@
-import { getModelConfig, getModelFromConfig } from "@/agent/utils";
+import {
+  createContextDocumentMessages,
+  getModelConfig,
+  getModelFromConfig,
+  isUsingO1MiniModel,
+} from "@/agent/utils";
 import { BaseLanguageModelInput } from "@langchain/core/language_models/base";
 import { AIMessageChunk } from "@langchain/core/messages";
 import { RunnableBinding } from "@langchain/core/runnables";
@@ -35,21 +40,21 @@ export const updateHighlightedText = async (
   state: typeof OpenCanvasGraphAnnotation.State,
   config: LangGraphRunnableConfig
 ): Promise<OpenCanvasGraphReturnType> => {
-  const { modelProvider } = getModelConfig(config);
+  const { modelProvider, modelName } = getModelConfig(config);
   let model: RunnableBinding<
     BaseLanguageModelInput,
     AIMessageChunk,
     ConfigurableChatModelCallOptions
   >;
-  if (modelProvider.includes("openai")) {
-    // Custom model is OpenAI/Azure OpenAI
+  if (modelProvider.includes("openai") || modelName.includes("3-5-sonnet")) {
+    // Custom model is intelligent enough for updating artifacts
     model = (
       await getModelFromConfig(config, {
         temperature: 0,
       })
     ).withConfig({ runName: "update_highlighted_markdown" });
   } else {
-    // Custom model is not set to OpenAI/Azure OpenAI. Use GPT-4o
+    // Custom model is not intelligent enough for updating artifacts
     model = (
       await getModelFromConfig(
         {
@@ -92,11 +97,14 @@ export const updateHighlightedText = async (
     throw new Error("Expected a human message");
   }
 
+  const contextDocumentMessages = await createContextDocumentMessages(config);
+  const isO1MiniModel = isUsingO1MiniModel(config);
   const response = await model.invoke([
     {
-      role: "system",
+      role: isO1MiniModel ? "user" : "system",
       content: formattedPrompt,
     },
+    ...contextDocumentMessages,
     recentUserMessage,
   ]);
   const responseContent = response.content as string;