Add a vision bridge for text-only main models

README.md

@@ -98,9 +98,42 @@ Environment variables override saved config for a single run, handy in scripts a
 | `SETFREE_API_KEY` | gateway API key |
 | `SETFREE_MODEL` | model for this run |
 | `SETFREE_GATEWAY` | which saved gateway to use |
+| `SETFREE_VISION_MODEL` | vision model id; setting this enables the vision bridge |
+| `SETFREE_VISION_BASE_URL` | separate endpoint for the vision model (optional) |
+| `SETFREE_VISION_API_KEY` | API key for the vision model (optional) |
+| `SETFREE_VISION_OFF` | `1` hard-disables the bridge for a run |
+| `SETFREE_VISION_CONCURRENCY` | images captioned in parallel (default `8`) |
+| `SETFREE_VISION_IDLE_TIMEOUT` | exit the proxy after this long with no request (default: off) |
 
 Order of precedence: env vars, then saved config, then interactive setup (terminal only).
 
+## Vision bridge: a text-only model with eyes
+
+Some of the best models for long coding sessions are text-only. You pick them for the context window, then hit a wall the moment an image lands in the conversation — the model throws a "not multimodal" error and the turn wedges.
+
+The vision bridge fixes that when your gateway also serves a multimodal model. With it on, SetFree starts a tiny local proxy that the CLI talks to instead of the gateway directly. The proxy forwards everything unchanged **except** image content: each image is sent to your vision model, described, and replaced with a text caption before the request reaches the main model. The text model never sees a raw image block, so it never errors. Captions are cached to disk, so an image is described once ever, not once per turn.
+
+Enable it by naming a vision model (saved or via `SETFREE_VISION_MODEL`):
+
+```toml
+# in config.toml's [vision] table
+[vision]
+model = "qwen3.5"
+# base_url and api_key are optional; they default to your main gateway
+```
+
+or for a single run:
+
+```sh
+SETFREE_VISION_MODEL=qwen3.5 setfree claude
+```
+
+This is the one, deliberate exception to SetFree's "step aside, never sit in the request path" rule. It's strictly opt-in — with no vision model set, no proxy is started and nothing about the launch differs from before. The proxy runs on localhost, lives only as long as the CLI does, and never touches authentication or licensing. The CLI binary itself is still the one you installed, unmodified.
+
+The proxy's lifetime is the launch's: SetFree runs the CLI as a child while the bridge is on, so it stops the proxy when the CLI exits, and the proxy watches its own parent pid so that a SetFree killed outright doesn't leave it behind. There is deliberately no idle timeout by default — the CLI holds the proxy's address for the whole session, so a proxy that exited between turns would leave every following request refused until the CLI restarted. `SETFREE_VISION_IDLE_TIMEOUT` adds one back for an unattended manual run (`setfree vision-proxy`); `0` keeps it off.
+
+Two things the bridge can't do, because they'd require modifying the binary rather than proxying it: keep the original image for an on-demand "look closer" re-query, and let the text model ask the vision model about a specific detail it didn't caption well. The text model gets the caption and works from there. That's the tradeoff for staying out of the binary.
+
 ## Adding a CLI adapter
 
 Read `internal/adapters/claude/claude.go` or `internal/adapters/codex/codex.go`. Each is under 60 lines. To add your own:

internal/app/app.go

@@ -17,6 +17,7 @@ import (
 	"github.com/mindsdb/setfree/internal/terminal"
 	"github.com/mindsdb/setfree/internal/ui"
 	"github.com/mindsdb/setfree/internal/version"
+	"github.com/mindsdb/setfree/internal/vision"
 )
 
 // env holds everything a command needs, resolved once per run.
@@ -71,6 +72,10 @@ func Run(args []string) int {
 	case "usage":
 		maybeSelfUpdate()
 		return cmdUsage(args[1:])
+	case vision.ProxySubcommand:
+		// Internal: the launcher relaunches the setfree binary as the vision
+		// proxy. Not advertised; no self-update on this path.
+		return vision.Run(args[1:])
 	default:
 		maybeSelfUpdate()
 		return cmdLaunch(args[0], args[1:])

internal/app/launch.go

@@ -14,6 +14,7 @@ import (
 	"github.com/mindsdb/setfree/internal/launcher"
 	"github.com/mindsdb/setfree/internal/terminal"
 	"github.com/mindsdb/setfree/internal/ui"
+	"github.com/mindsdb/setfree/internal/vision"
 )
 
 // cmdLaunch is `setfree <cli> [args...]`.
@@ -80,8 +81,29 @@ func cmdLaunch(name string, passthrough []string) int {
 		}
 	}
 
+	// Vision bridge: when a vision model is configured, start a local proxy
+	// that captions image content with a separate multimodal model, and point
+	// the CLI at it instead of the gateway. The CLI binary is unchanged; it
+	// just talks to our proxy, which forwards everything else untouched. The
+	// proxy lives only as long as this launch, so the parent must stay alive
+	// to tear it down — which is why the vision path runs the CLI as a child
+	// (launcher.Run) instead of exec-replacing (launcher.Launch).
+	vcfg := vision.Resolve(e.settings, e.store, os.Getenv, config.GatewaySetting{BaseURL: resolved.Gateway.BaseURL}, resolved.Gateway.APIKey)
+	vres, err := vision.MaybeStart(context.Background(), resolved, vcfg)
+	if err != nil {
+		debugf("vision bridge: %v", err)
+		// A bridge that can't start shouldn't block the user: fall back to a
+		// normal launch against the gateway. Images will 400 as before, but
+		// text work proceeds.
+		vres.ProxyURL = ""
+	}
+	if vres.ProxyURL != "" {
+		resolved.Gateway.BaseURL = vres.ProxyURL
+	}
+
 	build, err := adapter.Build(os.Environ(), resolved)
 	if err != nil {
+		vres.Stop()
 		return fail(err)
 	}
 
@@ -96,10 +118,23 @@ func cmdLaunch(name string, passthrough []string) int {
 	}
 
 	argv := buildArgv(path, build.Args, passthrough)
+	opts := launcher.Options{Path: path, Args: argv, Env: build.Env}
 
-	code, err := launcher.Launch(launcher.Options{Path: path, Args: argv, Env: build.Env})
+	// Vision off → exec-replace as always (no parent process left behind).
+	// Vision on → run the CLI as a child so the proxy can be torn down on
+	// exit, then return the child's exit code.
+	if vres.ProxyURL == "" {
+		code, err := launcher.Launch(opts)
+		if err != nil {
+			debugf("exec of %s failed: %v", path, err)
+			return fail(fmt.Errorf("couldn't launch %s: %w", adapter.DisplayName(), err))
+		}
+		return code
+	}
+	code, err := launcher.Run(opts)
+	vres.Stop()
 	if err != nil {
-		debugf("exec of %s failed: %v", path, err)
+		debugf("run of %s failed: %v", path, err)
 		return fail(fmt.Errorf("couldn't launch %s: %w", adapter.DisplayName(), err))
 	}
 	return code

internal/config/settings.go

@@ -20,6 +20,22 @@ type Settings struct {
 	DefaultGateway string                    `toml:"default_gateway,omitempty"`
 	Gateways       map[string]GatewaySetting `toml:"gateways,omitempty"`
 	CLI            map[string]CLISetting     `toml:"cli,omitempty"`
+	// Vision, when its Model is set, enables the vision bridge: a local
+	// proxy that captions image content with a separate multimodal model so a
+	// text-only main model never receives a raw image block. Inert when
+	// unset. The bridge's API key lives in the secrets store under the
+	// "vision" name, not here.
+	Vision VisionSetting `toml:"vision,omitempty"`
+}
+
+// VisionSetting is the non-secret half of the vision-bridge config. The
+// bridge is off unless Model names a multimodal model to caption with.
+// BaseURL and the API key are optional; when unset they fall back to the
+// main gateway's, so the common case (one endpoint serving both a text and a
+// vision model) needs only the model id.
+type VisionSetting struct {
+	Model   string `toml:"model,omitempty"`
+	BaseURL string `toml:"base_url,omitempty"`
 }
 
 // GatewaySetting holds the non-secret half of a configured gateway.

internal/config/settings_test.go

@@ -43,6 +43,34 @@ func TestSaveLoad_RoundTrip(t *testing.T) {
 	}
 }
 
+func TestSaveLoad_VisionRoundTrip(t *testing.T) {
+	dir := t.TempDir()
+	s := &Settings{Version: CurrentVersion, DefaultGateway: "default"}
+	s.SetGatewayBaseURL("default", "https://gw.example.com")
+	s.Vision = VisionSetting{Model: "qwen3.5", BaseURL: "https://vision.example.com"}
+	if err := Save(dir, s); err != nil {
+		t.Fatalf("Save: %v", err)
+	}
+	loaded, err := Load(dir)
+	if err != nil {
+		t.Fatalf("Load: %v", err)
+	}
+	if loaded.Vision.Model != "qwen3.5" || loaded.Vision.BaseURL != "https://vision.example.com" {
+		t.Errorf("Vision = %+v, want model=qwen3.5 base=https://vision.example.com", loaded.Vision)
+	}
+}
+
+func TestLoad_EmptySettingsHasNoVision(t *testing.T) {
+	dir := t.TempDir()
+	loaded, err := Load(dir)
+	if err != nil {
+		t.Fatalf("Load: %v", err)
+	}
+	if loaded.Vision.Model != "" {
+		t.Errorf("fresh settings should have no vision model, got %q", loaded.Vision.Model)
+	}
+}
+
 func TestLoad_RejectsNewerSchemaVersion(t *testing.T) {
 	dir := t.TempDir()
 	future := &Settings{Version: CurrentVersion + 1}

internal/launcher/launcher.go

@@ -26,3 +26,18 @@ type Options struct {
 func Launch(opts Options) (exitCode int, err error) {
 	return launch(opts)
 }
+
+// Run spawns the process described by opts as a child and waits for it,
+// returning its exit code. Unlike Launch (which exec-replaces on Unix), Run
+// keeps the SetFree process alive as the parent — so a caller that started
+// sidecar processes (e.g. the vision proxy) can tear them down when the child
+// exits. Signals to the parent are forwarded to the child so the launched
+// CLI behaves like it was invoked directly.
+//
+// This is the path the vision bridge uses: the proxy must live exactly as
+// long as the CLI, which means SetFree has to stay around to kill it. Every
+// other launch still uses Launch (exec-replace), so the common case is
+// unchanged.
+func Run(opts Options) (exitCode int, err error) {
+	return run(opts)
+}

internal/launcher/launcher_unix.go

@@ -2,7 +2,12 @@
 
 package launcher
 
-import "syscall"
+import (
+	"os"
+	"os/exec"
+	"os/signal"
+	"syscall"
+)
 
 func launch(opts Options) (int, error) {
 	// syscall.Exec replaces this process's image with the target binary.
@@ -14,3 +19,57 @@ func launch(opts Options) (int, error) {
 	// Reached only on failure; on success the process image is gone.
 	return 0, err
 }
+
+// run spawns opts as a foreground child in the same process group and waits
+// for it, forwarding termination signals so the child behaves like a direct
+// invocation. The child shares the foreground process group, so terminal-
+// generated signals (Ctrl-C in cooked mode, SIGWINCH on resize) reach it
+// directly; this parent only needs to (a) not die on Ctrl-C before the child
+// reports its exit code, and (b) forward externally-sent signals (SIGTERM,
+// SIGHUP, SIGQUIT from `kill`) that are delivered to this process alone.
+func run(opts Options) (int, error) {
+	cmd := exec.Command(opts.Path)
+	if len(opts.Args) > 0 {
+		cmd.Args = opts.Args
+	}
+	cmd.Env = opts.Env
+	cmd.Stdin = os.Stdin
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+
+	if err := cmd.Start(); err != nil {
+		return 0, err
+	}
+
+	// SIGINT (Ctrl-C) is delivered to the whole foreground process group, so
+	// the child already receives it; draining it here just keeps this parent
+	// alive to report the child's real exit code. Forward the rest.
+	sigCh := make(chan os.Signal, 4)
+	signal.Notify(sigCh, os.Interrupt, syscall.SIGTERM, syscall.SIGHUP, syscall.SIGQUIT)
+	done := make(chan struct{})
+	go func() {
+		for {
+			select {
+			case s := <-sigCh:
+				if s == os.Interrupt {
+					continue // child got it directly via the shared pgroup
+				}
+				_ = cmd.Process.Signal(s)
+			case <-done:
+				return
+			}
+		}
+	}()
+
+	err := cmd.Wait()
+	signal.Stop(sigCh)
+	close(done)
+
+	if err == nil {
+		return 0, nil
+	}
+	if exitErr, ok := err.(*exec.ExitError); ok {
+		return exitErr.ExitCode(), nil
+	}
+	return 0, err
+}

internal/launcher/launcher_windows.go

@@ -46,3 +46,11 @@ func launch(opts Options) (int, error) {
 	}
 	return 0, err
 }
+
+// run is identical to launch on Windows: Windows can't replace a process in
+// place, so launch already spawns-and-waits. The Run entry point exists for
+// callers that need the parent to stay alive (the vision bridge), and on
+// Windows that's the only mode there is.
+func run(opts Options) (int, error) {
+	return launch(opts)
+}