experimental/bundletest: add developer testing workflow

.nextchanges/cli/6646.md

@@ -0,0 +1 @@
+* Add `databricks bundle test` for running BundleTest workflows. ([#6646](https://github.com/databricks/cli/pull/6646))

cmd/bundle/bundle.go

@@ -29,6 +29,7 @@ Online documentation: https://docs.databricks.com/en/dev-tools/bundles/index.htm
 	cmd.AddCommand(newDeployCommand())
 	cmd.AddCommand(newDestroyCommand())
 	cmd.AddCommand(newRunCommand())
+	cmd.AddCommand(newTestCommand())
 	cmd.AddCommand(newSchemaCommand())
 	cmd.AddCommand(newSyncCommand())
 	cmd.AddCommand(newValidateCommand())

cmd/bundle/test.go

@@ -0,0 +1,58 @@
+package bundle
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+
+	"github.com/databricks/cli/libs/execv"
+	"github.com/spf13/cobra"
+)
+
+var (
+	bundleTestLookPath = exec.LookPath
+	bundleTestExecv    = execv.Execv
+)
+
+func newTestCommand() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "test [bundletest-args...]",
+		Short: "Run tests for a bundle with bundletest",
+		Long: `Run tests for a bundle with the experimental bundletest runner.
+
+All arguments are passed directly to bundletest. Arguments that bundletest does
+not consume are forwarded to pytest, including arguments after --.
+
+Examples:
+  databricks bundle test --local -v
+  databricks bundle test --local --changed --base origin/main
+  databricks bundle test --cloud --profile dev --warehouse-id abc123
+  databricks bundle test --local -- -k transform_orders`,
+		Args:               cobra.ArbitraryArgs,
+		DisableFlagParsing: true,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return executeBundleTest(args)
+		},
+	}
+
+	return cmd
+}
+
+func executeBundleTest(args []string) error {
+	runner, err := bundleTestLookPath("bundletest")
+	if err != nil {
+		return fmt.Errorf("cannot find bundletest on PATH; install it with `uv pip install -e /path/to/databricks-cli/experimental/bundletest`: %w", err)
+	}
+
+	argv := make([]string, 1, len(args)+1)
+	argv[0] = runner
+	argv = append(argv, args...)
+	err = bundleTestExecv(execv.Options{
+		Args: argv,
+		Env:  os.Environ(),
+	})
+	if err != nil {
+		return fmt.Errorf("failed to run bundletest: %w", err)
+	}
+	return nil
+}

cmd/bundle/test_test.go

@@ -0,0 +1,103 @@
+package bundle
+
+import (
+	"errors"
+	"os/exec"
+	"testing"
+
+	"github.com/databricks/cli/libs/execv"
+	"github.com/spf13/cobra"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func stubBundleTestProcess(t *testing.T, lookPath func(string) (string, error), execute func(execv.Options) error) {
+	originalLookPath := bundleTestLookPath
+	originalExecv := bundleTestExecv
+	bundleTestLookPath = lookPath
+	bundleTestExecv = execute
+	t.Cleanup(func() {
+		bundleTestLookPath = originalLookPath
+		bundleTestExecv = originalExecv
+	})
+}
+
+func executeTestCommand(t *testing.T, args ...string) error {
+	root := &cobra.Command{Use: "databricks", SilenceErrors: true, SilenceUsage: true}
+	bundle := &cobra.Command{Use: "bundle"}
+	root.AddCommand(bundle)
+	bundle.AddCommand(newTestCommand())
+	root.SetArgs(append([]string{"bundle", "test"}, args...))
+	return root.ExecuteContext(t.Context())
+}
+
+func TestBundleTestForwardsArgumentsUnchanged(t *testing.T) {
+	t.Setenv("BUNDLETEST_ADAPTER_TEST", "present")
+
+	var options execv.Options
+	stubBundleTestProcess(t,
+		func(name string) (string, error) {
+			assert.Equal(t, "bundletest", name)
+			return "/resolved/bundletest", nil
+		},
+		func(got execv.Options) error {
+			options = got
+			return nil
+		},
+	)
+
+	err := executeTestCommand(t,
+		"--local",
+		"--changed",
+		"--base", "origin/main",
+		"-k", "transform_orders",
+		"--",
+		"tests/test_job.py::test_transform",
+	)
+	require.NoError(t, err)
+	assert.Equal(t, []string{
+		"/resolved/bundletest",
+		"--local",
+		"--changed",
+		"--base", "origin/main",
+		"-k", "transform_orders",
+		"--",
+		"tests/test_job.py::test_transform",
+	}, options.Args)
+	assert.Contains(t, options.Env, "BUNDLETEST_ADAPTER_TEST=present")
+	assert.Empty(t, options.Dir)
+}
+
+func TestBundleTestRunnerNotFound(t *testing.T) {
+	stubBundleTestProcess(t,
+		func(string) (string, error) {
+			return "", exec.ErrNotFound
+		},
+		func(execv.Options) error {
+			return errors.New("execv should not be called")
+		},
+	)
+
+	err := executeTestCommand(t, "--local")
+	require.Error(t, err)
+	assert.ErrorIs(t, err, exec.ErrNotFound)
+	assert.ErrorContains(t, err, "cannot find bundletest on PATH")
+	assert.ErrorContains(t, err, "experimental/bundletest")
+}
+
+func TestBundleTestRunnerStartError(t *testing.T) {
+	startErr := errors.New("start failed")
+	stubBundleTestProcess(t,
+		func(string) (string, error) {
+			return "/resolved/bundletest", nil
+		},
+		func(execv.Options) error {
+			return startErr
+		},
+	)
+
+	err := executeTestCommand(t, "--local")
+	require.Error(t, err)
+	assert.ErrorIs(t, err, startErr)
+	assert.ErrorContains(t, err, "failed to run bundletest")
+}

experimental/bundletest/README.md

@@ -2,6 +2,36 @@
 
 Experimental, pytest-style **isolation testing** for Databricks Asset Bundles (DABs).
 
+## Quick start
+
+Install the package in a bundle project, generate a starter test, and run locally:
+
+```sh
+uv pip install -e /path/to/cli/experimental/bundletest
+databricks bundle test init
+databricks bundle test --local
+```
+
+`bundletest init` finds the nearest `databricks.yml`, chooses a declared resource, and
+creates `tests/conftest.py` plus a marked `tests/test_bundle.py`. It refuses to overwrite
+either file unless `--force` is supplied.
+
+Every local run begins with a support report, so unsupported tasks are visible before
+pytest starts:
+
+```text
+bundletest local support: orders
+
+[LOCAL ] jobs.transform_orders/transform: runs src/transform_orders.sql
+[LOCAL ] jobs.aggregate_orders/aggregate: runs src/aggregate_orders.sql
+[CLOUD ] jobs.score_model/score: notebook_task requires a Databricks workspace
+[CONFIG] 33 non-job resources: configuration assertions only
+
+summary: 2 local, 1 cloud-only, 33 config-only
+```
+
+Use `databricks bundle test --support-only` for the report without a test run.
+
 Unit tests answer "does my function return the right value?" `bundletest` answers the
 next question up: **"does my deployed bundle resource actually produce the right table?"**
 That class of bug — a job wired to the wrong upstream, a renamed table, a transform
@@ -35,6 +65,25 @@ We isolate by substituting the component's **data-boundary neighbors**, never by
 the component's own output — faking the thing under test is a tautology that catches
 nothing.
 
+Job runs are checked by default. A failed task raises an assertion with the resource,
+task, source file, backend, run ID when available, and the original error:
+
+```text
+bundle job 'transform_orders' failed
+  task: transform
+  source: src/transform_orders.sql
+  backend: local
+  error: Catalog Error: Table raw_orders does not exist
+  use check=False to inspect an expected failure
+```
+
+Expected-failure tests can opt out explicitly:
+
+```python
+result = env.run_job("transform_orders", check=False)
+assert not result.succeeded
+```
+
 ## Two backends, one seam
 
 The same test runs against either backend, chosen by the `BUNDLETEST_BACKEND` env var:
@@ -84,21 +133,49 @@ the CLI repo are required in addition to Python.
 ```sh
 uv venv --python 3.12
 uv pip install -e ".[dev]"
-uv run pytest -v
+databricks bundle test --bundle examples/orders_bundle --local -v
 ```
 
+Arguments that `bundletest` does not consume are passed to pytest, so `-k`, `-x`, `-v`,
+node IDs, and plugins continue to work normally. Use `--no-support-report` for compact CI
+output.
+
+### Run tests affected by a change
+
+Mark each test with the resources it exercises:
+
+```python
+@pytest.mark.bundle_resource("jobs.transform_orders")
+def test_transform_dedupes(env): ...
+```
+
+Then select tests from the Git diff:
+
+```sh
+databricks bundle test --local --changed --base origin/main
+```
+
+`bundletest` maps changed paths such as `src/transform_orders.sql` back to the bundle
+resources that reference them. It runs tests with matching `bundle_resource` markers and
+always includes changed test files. A YAML change runs the complete suite because it can
+alter resource wiring, variables, or targets. The selection includes committed, staged,
+unstaged, and untracked files.
+
 ### Run it on cloud
 
 The cloud backend deploys to a real workspace. Example fixture `examples/cloud_orders/` contains two SQL jobs, a managed volume, and a file_path dashboard under `main.bundletest_cloud`:
 
 ```sh
-export BUNDLETEST_BACKEND=cloud
-export BUNDLETEST_PROFILE=<profile>              # from ~/.databrickscfg
-export BUNDLETEST_WAREHOUSE_ID=<sql-warehouse-id>  # used for seeding + assertion queries
 export BUNDLE_VAR_warehouse_id=<sql-warehouse-id>
-uv run --extra dev pytest examples/cloud_orders
+databricks bundle test --cloud \
+  --bundle examples/cloud_orders \
+  --profile <profile> \
+  --warehouse-id <sql-warehouse-id>
 ```
 
+The command requires `--profile` for cloud runs; it never selects a Databricks profile
+implicitly. Add `--target <target>` when the bundle has a dedicated test target.
+
 (`examples/orders_bundle/` is local static-config only, not deployable to cloud.)
 
 Seeded tables and job runs are real and cost money, so unlike the local backend (a fresh

experimental/bundletest/examples/cloud_orders/tests/test_cloud_e2e.py

@@ -8,6 +8,7 @@
 import pytest
 
 
+@pytest.mark.bundle_resource("jobs.transform_orders", "jobs.aggregate_orders")
 def test_bronze_to_silver_to_gold(env, schema):
     env.seed(
         f"{schema}.raw_orders",
@@ -33,17 +34,20 @@ def test_bronze_to_silver_to_gold(env, schema):
 
 
 @pytest.mark.cloud_only
+@pytest.mark.bundle_resource("jobs.transform_orders")
 def test_price_type_is_databricks_decimal(env, schema):
     env.seed(f"{schema}.raw_orders", [{"order_id": 1, "total_price": 10.0}])
     env.run_job("transform_orders")
     assert env.table(f"{schema}.orders").schema["total_price"] == "decimal(10,2)"
 
 
+@pytest.mark.bundle_resource("jobs.transform_orders")
 def test_job_is_wired_to_its_sql(env):
     job = env.backend.get_resource("jobs", "transform_orders")
     assert job["tasks"][0]["sql_task"]["file"]["path"].endswith("transform_orders.sql")
 
 
+@pytest.mark.bundle_resource("dashboards.orders_overview")
 def test_dashboard_source_tables_from_file_path(env, schema):
     # The dashboard is defined by file_path, not inline, yet source_tables() still resolves:
     # `bundle summary` inlines the file's serialized form at config-load, so get_resource has it.
@@ -52,6 +56,7 @@ def test_dashboard_source_tables_from_file_path(env, schema):
     assert dashboard.source_tables() == [f"{schema}.order_summary"]
 
 
+@pytest.mark.bundle_resource("volumes.raw_data")
 def test_uploaded_csv_is_readable(env, tmp_path):
     csv = tmp_path / "orders.csv"
     csv.write_text("order_id,total_price\n1,10.0\n2,5.0\n")
@@ -65,6 +70,7 @@ def test_uploaded_csv_is_readable(env, tmp_path):
 
 
 @pytest.mark.cloud_only
+@pytest.mark.bundle_resource("jobs.transform_orders")
 def test_deployed_job_carries_server_filled_fields(env):
     # get_deployed reads the workspace's stored object, so it carries values the server filled
     # in or normalized that our databricks.yml never declared — what get_resource (the declared

experimental/bundletest/examples/orders_bundle/tests/test_job_config.py

@@ -16,7 +16,10 @@
 the typed handles (env.pipeline, env.dashboard, ...) add resource-specific accessors on top.
 """
 
+import pytest
 
+
+@pytest.mark.bundle_resource("jobs.transform_orders")
 def test_job_is_wired_to_its_sql(env):
     job = env.backend.get_resource("jobs", "transform_orders")
     assert job["name"] == "transform_orders"

experimental/bundletest/examples/orders_bundle/tests/test_job_sql.py

@@ -19,6 +19,7 @@
 import pytest
 
 
+@pytest.mark.bundle_resource("jobs.transform_orders")
 def test_transform_dedupes(env):
     env.seed(
         "shop.bronze.raw_orders",
@@ -43,6 +44,7 @@ def test_transform_dedupes(env):
 
 
 @pytest.mark.cloud_only
+@pytest.mark.bundle_resource("jobs.transform_orders")
 def test_price_type_is_databricks_decimal(env):
     env.seed("shop.bronze.raw_orders", [{"order_id": 1, "total_price": 10.0}])
     env.run_job("transform_orders")

experimental/bundletest/examples/orders_bundle/tests/test_pipeline_end_to_end.py

@@ -12,7 +12,10 @@
 - automatic dependency ordering (here the test sequences run_job calls itself)
 """
 
+import pytest
 
+
+@pytest.mark.bundle_resource("jobs.transform_orders", "jobs.aggregate_orders")
 def test_bronze_to_silver_to_gold(env):
     env.seed(
         "shop.bronze.raw_orders",

experimental/bundletest/pyproject.toml

@@ -11,14 +11,18 @@ requires-python = ">=3.10"
 license = "Apache-2.0"
 dependencies = [
     "duckdb>=1.0",
+    "pytest>=7.0",
     "pyyaml>=6.0",
 ]
 
 [project.optional-dependencies]
 # The cloud backend talks to a real workspace through the Databricks SDK. Kept optional and
 # lazily imported so a local-only install (and its DuckDB backend) never pulls it in.
 cloud = ["databricks-sdk>=0.40"]
-dev = ["pytest>=7.0", "databricks-sdk>=0.40"]
+dev = ["databricks-sdk>=0.40"]
+
+[project.scripts]
+bundletest = "bundletest.cli:main"
 
 [project.entry-points.pytest11]
 bundletest = "bundletest.pytest_plugin"