feat: tool assisted add data journey

runtime/ai/ai.go

@@ -76,6 +76,7 @@ func NewRunner(rt *runtime.Runtime, activity *activity.Client) *Runner {
 	RegisterTool(r, &ListBuckets{Runtime: rt})
 	RegisterTool(r, &ListBucketObjects{Runtime: rt})
 
+	RegisterTool(r, &RequestConnectorFields{Runtime: rt})
 	RegisterTool(r, &Navigate{})
 
 	return r

runtime/ai/develop_file.go

@@ -127,6 +127,7 @@ func (t *DevelopFile) Handler(ctx context.Context, args *DevelopFileArgs) (*Deve
 			ListTablesName,
 			ShowTableName,
 			QuerySQLName,
+			RequestConnectorFieldsName,
 		},
 		MaxIterations: 10,
 		UnwrapCall:    true,

runtime/ai/developer_agent.go

@@ -97,6 +97,7 @@ func (t *DeveloperAgent) Handler(ctx context.Context, args *DeveloperAgentArgs)
 			ShowTableName,
 			QuerySQLName,
 			DevelopFileName,
+			RequestConnectorFieldsName,
 			NavigateName,
 		},
 		MaxIterations: 20,

runtime/ai/instructions/data/resources/connector.md

@@ -42,6 +42,18 @@ aws_secret_access_key: "{{ .env.aws_secret_access_key }}"
 
 NOTE: Some legacy projects use the deprecated `.vars` instead of `.env`.
 
+### Interactive UI (Rill app): requesting missing connector fields
+
+When you are running in the Rill UI and a connector cannot be completed from the user’s message alone (for example ClickHouse at `localhost:9000` without credentials), or **`project_status` / `write_file` / reconcile errors** show authentication or configuration failures, call the tool **`request_connector_fields`**.
+
+Pass:
+
+- **`driver`**: the connector driver (e.g. `clickhouse`).
+- **`missing_fields`**: snake_case YAML property keys the user still must supply (e.g. `username`, `password`), inferred from this document, the user’s request, and error messages—not from inventing new keys.
+- Optionally **`message`**, **`related_errors`** (short excerpts from tool errors), and **`connector_path`** if known.
+
+The UI may intercept this tool call to show forms and write secrets to `.env` using `{{ .env.KEY }}` placeholders in the connector YAML. Prefer this over asking for passwords in plain chat text when credentials are required.
+
 ### Managed connectors
 
 OLAP connectors can be provisioned automatically by Rill using `managed: true`. This is supported for `duckdb` and `clickhouse` drivers:

runtime/ai/request_connector_fields.go

@@ -0,0 +1,95 @@
+package ai
+
+import (
+	"context"
+	"fmt"
+	"regexp"
+	"strings"
+
+	"github.com/modelcontextprotocol/go-sdk/mcp"
+	"github.com/rilldata/rill/runtime"
+)
+
+const RequestConnectorFieldsName = "request_connector_fields"
+
+// connectorFieldNameRe matches typical top-level connector YAML keys (snake_case, ASCII).
+var connectorFieldNameRe = regexp.MustCompile(`^[a-z][a-z0-9_]*$`)
+
+type RequestConnectorFields struct {
+	Runtime *runtime.Runtime
+}
+
+var _ Tool[*RequestConnectorFieldsArgs, *RequestConnectorFieldsResult] = (*RequestConnectorFields)(nil)
+
+// RequestConnectorFieldsArgs is filled by the LLM. The runtime does not derive missing fields from JSON Schema;
+// the client uses this payload to show connector forms (e.g. secrets to .env).
+type RequestConnectorFieldsArgs struct {
+	Driver        string   `json:"driver" jsonschema:"Connector driver name (e.g. clickhouse, s3, postgres)."`
+	MissingFields []string `json:"missing_fields" jsonschema:"YAML property keys still needed from the user (e.g. username, password)."`
+	Message       string   `json:"message,omitempty" jsonschema:"Optional short explanation for the end user."`
+	ConnectorPath string   `json:"connector_path,omitempty" jsonschema:"Optional connector resource path if known."`
+}
+
+// RequestConnectorFieldsResult echoes the handoff for the Rill UI and tests.
+type RequestConnectorFieldsResult struct {
+	Driver        string   `json:"driver"`
+	MissingFields []string `json:"missing_fields"`
+	Message       string   `json:"message,omitempty"`
+	ConnectorPath string   `json:"connector_path,omitempty"`
+}
+
+func (t *RequestConnectorFields) Spec() *mcp.Tool {
+	return &mcp.Tool{
+		Name:        RequestConnectorFieldsName,
+		Title:       "Request connector fields",
+		Description: "UI handoff: request that the user supply specific connector YAML fields (often credentials). Use when the connector setup is incomplete or reconcile errors indicate missing auth or config. Pass driver and missing_fields you infer from instructions, the user message, and tool errors. The Rill UI may intercept this call to show forms and write secrets to .env.",
+		Annotations: &mcp.ToolAnnotations{
+			DestructiveHint: boolPtr(false),
+			IdempotentHint:  true,
+			OpenWorldHint:   boolPtr(false),
+			ReadOnlyHint:    true,
+		},
+		Meta: map[string]any{
+			"openai/toolInvocation/invoking": "Requesting connector fields...",
+			"openai/toolInvocation/invoked":  "Connector fields requested",
+		},
+	}
+}
+
+func (t *RequestConnectorFields) CheckAccess(ctx context.Context) (bool, error) {
+	return checkDeveloperAccess(ctx, t.Runtime, true)
+}
+
+func (t *RequestConnectorFields) Handler(ctx context.Context, args *RequestConnectorFieldsArgs) (*RequestConnectorFieldsResult, error) {
+	driver := strings.TrimSpace(args.Driver)
+	if driver == "" {
+		return nil, fmt.Errorf("driver is required")
+	}
+
+	missing := make([]string, 0, len(args.MissingFields))
+	seen := make(map[string]struct{})
+	for _, raw := range args.MissingFields {
+		f := strings.TrimSpace(raw)
+		if f == "" {
+			continue
+		}
+		if !connectorFieldNameRe.MatchString(f) {
+			return nil, fmt.Errorf("invalid missing_fields entry %q: use snake_case keys like username or aws_access_key_id", raw)
+		}
+		if _, ok := seen[f]; ok {
+			continue
+		}
+		seen[f] = struct{}{}
+		missing = append(missing, f)
+	}
+	if len(missing) == 0 {
+		return nil, fmt.Errorf("missing_fields must contain at least one non-empty field key")
+	}
+
+	return &RequestConnectorFieldsResult{
+		Driver:        driver,
+		MissingFields: missing,
+		Message:       strings.TrimSpace(args.Message),
+		ConnectorPath: strings.TrimSpace(args.ConnectorPath),
+	}, nil
+}

runtime/ai/request_connector_fields_test.go

@@ -0,0 +1,67 @@
+package ai_test
+
+import (
+	"testing"
+
+	"github.com/rilldata/rill/runtime/ai"
+	"github.com/rilldata/rill/runtime/testruntime"
+	"github.com/stretchr/testify/require"
+)
+
+func TestRequestConnectorFields(t *testing.T) {
+	rt, instanceID := testruntime.NewInstanceWithOptions(t, testruntime.InstanceOptions{
+		Files: map[string]string{
+			"rill.yaml": `olap_connector: duckdb`,
+		},
+	})
+	s := newSession(t, rt, instanceID)
+
+	t.Run("success", func(t *testing.T) {
+		var res *ai.RequestConnectorFieldsResult
+		_, err := s.CallTool(t.Context(), ai.RoleUser, ai.RequestConnectorFieldsName, &res, &ai.RequestConnectorFieldsArgs{
+			Driver: "clickhouse",
+			MissingFields: []string{
+				"username",
+				" password ",
+				"username",
+			},
+			Message:      "Need credentials for localhost:9000",
+			ResourceName: "clickhouse_local",
+		})
+		require.NoError(t, err)
+		require.NotNil(t, res)
+		require.Equal(t, "clickhouse", res.Driver)
+		require.Equal(t, []string{"username", "password"}, res.MissingFields)
+		require.Equal(t, "Need credentials for localhost:9000", res.Message)
+		require.Equal(t, "clickhouse_local", res.ResourceName)
+	})
+
+	t.Run("missing driver", func(t *testing.T) {
+		var res *ai.RequestConnectorFieldsResult
+		_, err := s.CallTool(t.Context(), ai.RoleUser, ai.RequestConnectorFieldsName, &res, &ai.RequestConnectorFieldsArgs{
+			MissingFields: []string{"password"},
+		})
+		require.Error(t, err)
+		require.Contains(t, err.Error(), "driver is required")
+	})
+
+	t.Run("missing fields", func(t *testing.T) {
+		var res *ai.RequestConnectorFieldsResult
+		_, err := s.CallTool(t.Context(), ai.RoleUser, ai.RequestConnectorFieldsName, &res, &ai.RequestConnectorFieldsArgs{
+			Driver:        "s3",
+			MissingFields: nil,
+		})
+		require.Error(t, err)
+		require.Contains(t, err.Error(), "missing_fields")
+	})
+
+	t.Run("invalid field key", func(t *testing.T) {
+		var res *ai.RequestConnectorFieldsResult
+		_, err := s.CallTool(t.Context(), ai.RoleUser, ai.RequestConnectorFieldsName, &res, &ai.RequestConnectorFieldsArgs{
+			Driver:        "s3",
+			MissingFields: []string{"AWS_KEY"},
+		})
+		require.Error(t, err)
+		require.Contains(t, err.Error(), "invalid missing_fields")
+	})
+}

web-common/src/features/add-data/form/AddDataFormStructure.svelte

@@ -33,10 +33,10 @@
   export let schema: MultiStepFormSchema | null;
   export let superFormsParams: ReturnType<typeof createConnectorForm>;
   export let labels = defaultFormLabels;
-  export let yamlPreview: string;
+  export let yamlPreview: string | undefined = undefined;
   export let step: AddDataState;
   export let onSave: (() => void) | undefined = undefined;
-  export let onBack: () => void | Promise<void>;
+  export let onBack: (() => void | Promise<void>) | undefined = undefined;
 
   $: ({ form, formId, tainted, submit, submitting, errors, enhance } =
     superFormsParams);
@@ -47,7 +47,8 @@
 
   $: ({ message, details } = getSubmitError($errors));
 
-  $: hideRightPannel = connectorDriver.name === "local_file";
+  $: hideRightPannel =
+    connectorDriver.name === "local_file" && yamlPreview !== undefined;
 
   $: formClass = getFormClass(step);
 
@@ -190,14 +191,16 @@
     <div
       class="w-full bg-surface-subtle border-t border-gray-200 p-6 flex justify-between gap-2"
     >
-      <Button
-        disabled={runningBackAction}
-        loading={runningBackAction}
-        onClick={() => void handleBack()}
-        type="tertiary"
-      >
-        Back
-      </Button>
+      {#if onBack}
+        <Button
+          disabled={runningBackAction}
+          loading={runningBackAction}
+          onClick={() => void handleBack()}
+          type="tertiary"
+        >
+          Back
+        </Button>
+      {/if}
 
       <div class="flex gap-2">
         {#if onSave && isSaveButtonEnabled}

web-common/src/features/add-data/manager/steps/connector.ts

@@ -49,6 +49,7 @@ export async function createConnector({
   formValues,
   validate,
   existingEnvBlob,
+  pathOverride,
 }: {
   runtimeClient: RuntimeClient;
   queryClient: QueryClient;
@@ -57,6 +58,7 @@ export async function createConnector({
   formValues: Record<string, unknown>;
   validate: boolean;
   existingEnvBlob: string | null;
+  pathOverride?: string;
 }) {
   await maybeInitProject(runtimeClient, connectorDriver);
 
@@ -72,10 +74,9 @@ export async function createConnector({
     : [];
 
   // Create connector file path outside try block for cleanup
-  const newConnectorFilePath = getFileAPIPathFromNameAndType(
-    connectorName,
-    EntityType.Connector,
-  );
+  const newConnectorFilePath =
+    pathOverride ??
+    getFileAPIPathFromNameAndType(connectorName, EntityType.Connector);
 
   try {
     // Capture original .env and compute updated contents up front

web-common/src/features/chat/core/messages/Messages.svelte

@@ -15,6 +15,7 @@
   import ThinkingBlock from "./thinking/ThinkingBlock.svelte";
   import WorkingBlock from "./working/WorkingBlock.svelte";
   import SimpleToolCallBlock from "@rilldata/web-common/features/chat/core/messages/simple-tool-call/SimpleToolCallBlock.svelte";
+  import RequestConnectorFieldsBlock from "@rilldata/web-common/features/chat/core/messages/request-connector-fields/RequestConnectorFieldsBlock.svelte";
 
   export let conversationManager: ConversationManager;
   export let layout: "sidebar" | "fullpage";
@@ -147,6 +148,12 @@
         <FileDiffBlock {block} {tools} />
       {:else if block.type === "simple-tool-call-block"}
         <SimpleToolCallBlock {block} {tools} />
+      {:else if block.type === "request-connector-fields-block"}
+        <RequestConnectorFieldsBlock
+          conversation={currentConversation}
+          {block}
+          {tools}
+        />
       {/if}
     {/each}
   {/if}

web-common/src/features/chat/core/messages/block-transform.ts

@@ -31,6 +31,7 @@ import {
 } from "./tools/tool-registry";
 import { shouldShowWorking, type WorkingBlock } from "./working/working-block";
 import type { SimpleToolCall } from "@rilldata/web-common/features/chat/core/messages/simple-tool-call/simple-tool-call.ts";
+import type { RequestConnectorFieldsBlock } from "@rilldata/web-common/features/chat/core/messages/request-connector-fields/request-connector-fields-block.ts";
 
 // =============================================================================
 // TYPES
@@ -42,7 +43,8 @@ export type Block =
   | ChartBlock
   | FileDiffBlock
   | WorkingBlock
-  | SimpleToolCall;
+  | SimpleToolCall
+  | RequestConnectorFieldsBlock;
 
 export type {
   ChartBlock,
@@ -51,6 +53,7 @@ export type {
   ThinkingBlock,
   WorkingBlock,
   SimpleToolCall,
+  RequestConnectorFieldsBlock,
 };
 
 // =============================================================================
@@ -76,6 +79,7 @@ export function transformToBlocks(
 
   // Accumulator for messages going into the current thinking block
   let thinkingMessages: V1Message[] = [];
+  let requestConnectorBlocks: RequestConnectorFieldsBlock[] = [];
 
   function flushThinking(isComplete: boolean): void {
     if (thinkingMessages.length > 0) {
@@ -102,6 +106,10 @@ export function transformToBlocks(
         const feedback =
           msg.role === "assistant" ? feedbackMap.get(msg.id!) : undefined;
         blocks.push(createTextBlock(msg, feedback));
+        if (requestConnectorBlocks.length > 0) {
+          blocks.push(...requestConnectorBlocks);
+          requestConnectorBlocks = [];
+        }
         break;
       }
 
@@ -111,8 +119,15 @@ export function transformToBlocks(
 
       case "block": {
         flushThinking(true);
-        const block = routing.config.createBlock?.(msg, resultMap.get(msg.id));
-        if (block) {
+        const block = routing.config.createBlock?.(
+          msg,
+          resultMap.get(msg.id),
+          messages,
+        );
+        if (!block) break;
+        if (block.type === "request-connector-fields-block") {
+          requestConnectorBlocks.push(block);
+        } else {
           blocks.push(block);
         }
         break;