experimental/bundletest: offline variable resolution for the local backend

.github/workflows/python_push.yml

@@ -74,6 +74,13 @@ jobs:
       - name: Checkout repository and submodules
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
 
+      # The local backend resolves bundle config by running the in-repo Go helper
+      # experimental/bundletest/cmd/offline-resolve, so the suite needs the Go toolchain.
+      - name: Setup Go
+        uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
+        with:
+          go-version-file: go.mod
+
       - name: Install uv
         uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
         with:

experimental/bundletest/README.md

@@ -64,13 +64,23 @@ The same test runs against either backend, chosen by the `BUNDLETEST_BACKEND` en
     (`main`/`temp`/`system`) can't be judged locally → `LocalUnsupported` → the test
     **skips with a reason**;
   - anything that runs and disagrees with an assertion → **red**.
+- **Config resolved by the CLI's own engine, online references skipped loudly.** Deploy
+  runs the real offline resolution (`cmd/offline-resolve`) — includes, target overrides,
+  presets, and `${var.*}`/`${bundle.*}` all resolve exactly as `bundle validate` renders
+  them, with no auth or network and no reimplementation. A reference only the workspace can
+  resolve — `${workspace.*}`, a `lookup` variable, or an unset variable — is
+  `LocalUnsupported` at the use site, never silently passed through.
 
 Assertions you *know* are cloud-only (Databricks type naming, SLA timing, permissions)
 can also be fenced explicitly with `@pytest.mark.cloud_only`, which skips them on any
 non-cloud backend.
 
 ## Run it
 
+The local backend resolves bundle config with the in-repo Go helper
+`cmd/offline-resolve`, so a Go toolchain (matching the repo's `go.mod`) and a checkout of
+the CLI repo are required in addition to Python.
+
 ```sh
 uv venv --python 3.12
 uv pip install -e ".[dev]"

experimental/bundletest/cmd/offline-resolve/main.go

@@ -0,0 +1,162 @@
+// Command offline-resolve loads a Declarative Automation Bundle and resolves its
+// configuration entirely offline — no workspace client, no auth — then prints the
+// resolved config as JSON on stdout.
+//
+// bundletest's local (DuckDB) backend subprocesses this instead of reimplementing
+// variable/include/target resolution in Python. It reuses the CLI's own mutators, so
+// includes, target overrides, presets, and ${var.*}/${bundle.*} references resolve
+// exactly as `databricks bundle validate` would.
+//
+// Deliberately offline: it applies only the mutators that need no workspace. In
+// particular it never runs PopulateCurrentUser (the first auth call) or
+// ResolveLookupVariables (needs the workspace). References only the workspace can
+// resolve are preserved for the caller to reject loudly at the use site: ${workspace.*}
+// stays literal, and a lookup or unset variable comes through as a sentinel marker (see
+// SentinelFormat).
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"os"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/config/loader"
+	"github.com/databricks/cli/bundle/config/mutator"
+	"github.com/databricks/cli/bundle/config/validate"
+	"github.com/databricks/cli/libs/diag"
+	"github.com/databricks/cli/libs/dyn"
+	"github.com/databricks/cli/libs/dyn/convert"
+	"github.com/databricks/cli/libs/logdiag"
+)
+
+// SentinelFormat encodes a variable that can't be resolved offline. The two %s are the
+// reason ("lookup" or "unset") and the variable name. The caller (bundletest's local
+// backend) matches this marker in the resolved output and rejects the referencing
+// resource loudly, since resolving it needs a workspace.
+const SentinelFormat = "__bundletest_unresolved__%s__%s__"
+
+// seedOfflineUnresolvable gives each variable that has no offline value a sentinel
+// default and drops any lookup, so the real resolver can complete instead of aborting.
+//
+// Two things would otherwise fail offline: SetVariables errors on a required variable
+// with no value (set_variables.go), and dynvar errors resolving a reference to a lookup
+// variable whose value only a workspace can supply. Assigning a sentinel *default* (not a
+// value, so a BUNDLE_VAR_* / variable-file override still wins) and clearing lookup (so
+// SetVariables uses the default rather than deferring to ResolveLookupVariables) lets
+// ${var.<name>} resolve to the sentinel, which the caller detects. Must run before SetVariables.
+type seedOfflineUnresolvable struct{}
+
+func (seedOfflineUnresolvable) Name() string { return "seedOfflineUnresolvable" }
+
+func (seedOfflineUnresolvable) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
+	err := b.Config.Mutate(func(root dyn.Value) (dyn.Value, error) {
+		for name, v := range b.Config.Variables {
+			var reason string
+			switch {
+			case v.Lookup != nil:
+				reason = "lookup"
+			case !v.HasValue() && !v.HasDefault():
+				reason = "unset"
+			default:
+				continue
+			}
+			sentinel := fmt.Sprintf(SentinelFormat, reason, name)
+			var err error
+			root, err = dyn.Set(root, "variables."+name+".default", dyn.V(sentinel))
+			if err != nil {
+				return dyn.InvalidValue, err
+			}
+		}
+		// Drop every variables.*.lookup: the only ones present are on vars we just seeded.
+		return dyn.Walk(root, func(p dyn.Path, v dyn.Value) (dyn.Value, error) {
+			if len(p) == 3 && p[0] == dyn.Key("variables") && p[2] == dyn.Key("lookup") {
+				return v, dyn.ErrDrop
+			}
+			return v, nil
+		})
+	})
+	return diag.FromErr(err)
+}
+
+// offlinePrefixes are the variable-reference prefixes that resolve without a workspace.
+// "workspace" is deliberately excluded: workspace fields are unset offline and would
+// otherwise resolve to empty strings, hiding references the caller must reject.
+var offlinePrefixes = []string{"bundle", "variables"}
+
+func run(ctx context.Context, path, target string) error {
+	b, err := bundle.Load(ctx, path)
+	if err != nil {
+		return err
+	}
+
+	selectTarget := mutator.SelectDefaultTarget()
+	if target != "" {
+		selectTarget = mutator.SelectTarget(target)
+	}
+
+	ctx = logdiag.InitContext(ctx)
+	logdiag.SetCollect(ctx, true)
+	bundle.ApplySeqContext(ctx, b,
+		// --- load phase (offline subset of mutator.DefaultMutators) ---
+		loader.EntryPoint(),
+		loader.ProcessRootIncludes(),
+		mutator.EnvironmentsToTargets(),
+		mutator.ComputeIdToClusterId(),
+		mutator.InitializeVariables(),
+		mutator.DefineDefaultTarget(),
+		selectTarget,
+		bundle.Mutator(seedOfflineUnresolvable{}),
+
+		// --- initialize phase, everything before the first auth call
+		// (PopulateCurrentUser) that is safe offline ---
+		mutator.RejectInternalResources(),
+		validate.AllResourcesHaveValues(),
+		validate.ValidateEngine(),
+		validate.Scripts(),
+		mutator.RewriteSyncPaths(),
+		mutator.SyncDefaultPath(),
+		mutator.SyncInferRoot(),
+		mutator.InitializeCache(),
+
+		// Variable resolution. SetVariables assigns values from BUNDLE_VAR_*, variable
+		// files, and defaults. The two ResolveVariableReferences* mutators are the real
+		// engine, restricted to offline prefixes so workspace/lookup/unset refs stay literal.
+		mutator.SetVariables(),
+		mutator.ResolveVariableReferencesInLookup(),
+		mutator.ResolveVariableReferencesWithoutResources(offlinePrefixes...),
+		mutator.ResolveVariableReferencesOnlyResources(offlinePrefixes...),
+	)
+
+	diags := logdiag.FlushCollected(ctx)
+	if diags.HasError() {
+		return fmt.Errorf("offline resolution failed: %w", diags.Error())
+	}
+
+	converted, err := convert.FromTyped(&b.Config, b.Config.Value())
+	if err != nil {
+		return err
+	}
+	buf, err := json.Marshal(converted.AsAny())
+	if err != nil {
+		return err
+	}
+	os.Stdout.Write(buf)
+	return nil
+}
+
+func main() {
+	target := flag.String("target", "", "bundle target to select (default target if empty)")
+	flag.Parse()
+	if flag.NArg() != 1 {
+		fmt.Fprintln(os.Stderr, "usage: offline-resolve [--target NAME] <bundle-path>")
+		os.Exit(1)
+	}
+
+	if err := run(context.Background(), flag.Arg(0), *target); err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+}

experimental/bundletest/examples/orders_bundle/databricks.yml

@@ -12,6 +12,9 @@ bundle:
 variables:
   warehouse_id:
     description: SQL warehouse the transform runs on
+    # A default so ${var.warehouse_id} resolves offline; the local backend never contacts a
+    # warehouse, so the value is only a placeholder (real runs override via BUNDLE_VAR_* or target).
+    default: sql-warehouse-placeholder
 
 resources:
   jobs:

experimental/bundletest/src/bundletest/backends/duckdb.py

@@ -20,16 +20,17 @@
 
 from __future__ import annotations
 
+import json
 import os
 import re
 import shutil
+import subprocess
 import tempfile
 import time
 from pathlib import Path
 from typing import Any
 
 import duckdb
-import yaml
 
 from bundletest.backend import LocalUnsupported, RunResult
 
@@ -68,6 +69,66 @@ def _first_line(err: Exception) -> str:
     return str(err).splitlines()[0] if str(err) else type(err).__name__
 
 
+# Bundle config is resolved offline by the CLI's own engine (cmd/offline-resolve), which
+# reuses the real load/target/variable mutators — no reimplementation here. References only a
+# workspace can resolve come back two ways: ${workspace.*}/${resources.*} stay literal ($${...}
+# is the escape, so a ${ not preceded by $ is a real reference), and a lookup or unset variable
+# comes back as a sentinel marker. Both are rejected loudly at the use site.
+_VAR_REF = re.compile(r"(?<!\$)\$\{[^}]+\}")
+# Must match SentinelFormat in cmd/offline-resolve/main.go.
+_SENTINEL = re.compile(r"__bundletest_unresolved__(lookup|unset)__([A-Za-z_][\w-]*)__")
+
+# Package path of the offline resolver, relative to the repo root (the module containing go.mod).
+_OFFLINE_RESOLVE_PKG = "./experimental/bundletest/cmd/offline-resolve"
+
+
+def _repo_root() -> Path:
+    for p in Path(__file__).resolve().parents:
+        if (p / "go.mod").exists():
+            return p
+    raise RuntimeError("could not locate the repo root (go.mod) for the offline-resolve helper")
+
+
+def _resolve_config(bundle_path: str) -> dict[str, Any]:
+    """Load and resolve the bundle at ``bundle_path`` offline via the Go helper, returning the
+    resolved config as a dict."""
+    result = subprocess.run(
+        ["go", "run", _OFFLINE_RESOLVE_PKG, bundle_path],
+        cwd=_repo_root(),
+        capture_output=True,
+        text=True,
+    )
+    if result.returncode != 0:
+        raise RuntimeError(f"offline bundle resolution failed:\n{result.stderr.strip()}")
+    return json.loads(result.stdout)
+
+
+def _unresolved_in_str(s: str) -> str | None:
+    """A message describing why ``s`` can't be resolved locally (a sentinel-marked variable or a
+    residual ${...} reference), or None if it is fully resolved."""
+    m = _SENTINEL.search(s)
+    if m:
+        reason, name = m.group(1), m.group(2)
+        if reason == "lookup":
+            return f"variable {name!r} is a lookup variable that needs a workspace to resolve"
+        return f"variable {name!r} has no value offline — set BUNDLE_VAR_{name} or run on the cloud backend"
+    m = _VAR_REF.search(s)
+    if m:
+        return f"{m.group(0)} needs a workspace to resolve"
+    return None
+
+
+def _unresolved(node: Any) -> str | None:
+    """First offline-unresolvable reference anywhere in the subtree, as a message, or None."""
+    if isinstance(node, dict):
+        return next((r for r in map(_unresolved, node.values()) if r), None)
+    if isinstance(node, list):
+        return next((r for r in map(_unresolved, node) if r), None)
+    if isinstance(node, str):
+        return _unresolved_in_str(node)
+    return None
+
+
 class _Task:
     """One task of a job, as declared in databricks.yml."""
 
@@ -91,8 +152,7 @@ def __init__(self) -> None:
     # --- lifecycle ---
     def deploy(self, bundle_path: str) -> None:
         self._bundle_path = bundle_path
-        path = os.path.join(bundle_path, "databricks.yml")
-        self._config = yaml.safe_load(Path(path).read_text()) if os.path.exists(path) else {}
+        self._config = _resolve_config(bundle_path)
         self._jobs = self._extract_jobs(self._config)
 
     def teardown(self) -> None:
@@ -126,6 +186,12 @@ def run_job(self, name: str, params: dict[str, Any] | None = None) -> RunResult:
                         f"job {name!r} task {task.key!r} is a {task.kind} task; the "
                         f"local backend runs sql_task only — run it on the cloud backend"
                     )
+                if task.sql_file:
+                    reason = _unresolved_in_str(task.sql_file)
+                    if reason is not None:
+                        raise LocalUnsupported(
+                            f"job {name!r} task {task.key!r} sql path can't be resolved locally: {reason}"
+                        )
                 sql = Path(self._bundle_path, task.sql_file).read_text()
                 self._prepare_namespaces(sql)
                 for statement in _split_statements(sql):
@@ -154,7 +220,14 @@ def table_schema(self, fqn: str) -> dict[str, str]:
 
     # --- control plane ---
     def get_resource(self, kind: str, name: str) -> dict[str, Any]:
-        return self._config.get("resources", {}).get(kind, {})[name]
+        cfg = self._config.get("resources", {}).get(kind, {})[name]
+        reason = _unresolved(cfg)
+        if reason is not None:
+            raise LocalUnsupported(
+                f"resource {kind}.{name} can't be introspected locally: {reason}; "
+                f"use the cloud backend"
+            )
+        return cfg
 
     def put_file(self, dst: str, src: str) -> None:
         if not os.path.exists(src):