Validate serverless pipeline cluster settings

.nextchanges/bundles/validate-serverless-pipeline-clusters.md

@@ -0,0 +1 @@
+* Reject pipeline cluster settings when serverless compute is enabled during bundle validation, planning, and deployment. ([#6662](https://github.com/databricks/cli/pull/6662))

acceptance/bundle/validate/pipeline_serverless_clusters/databricks.yml

@@ -0,0 +1,39 @@
+bundle:
+  name: pipeline-serverless-clusters
+
+targets:
+  invalid:
+    resources:
+      pipelines:
+        my_pipeline:
+          name: my-pipeline
+          serverless: true
+          clusters:
+            - label: default
+              num_workers: 1
+
+  classic:
+    resources:
+      pipelines:
+        explicit:
+          name: explicit-classic
+          serverless: false
+          clusters:
+            - label: default
+              num_workers: 1
+        implicit:
+          name: implicit-classic
+          clusters:
+            - label: default
+              num_workers: 1
+
+  serverless:
+    resources:
+      pipelines:
+        omitted_clusters:
+          name: omitted-clusters
+          serverless: true
+        empty_clusters:
+          name: empty-clusters
+          serverless: true
+          clusters: []

acceptance/bundle/validate/pipeline_serverless_clusters/output.txt

@@ -0,0 +1,52 @@
+
+=== Reject cluster settings with serverless compute before deployment
+>>> [CLI] bundle validate -t invalid
+Error: Cannot configure clusters for a serverless pipeline
+  at resources.pipelines.my_pipeline.clusters
+  in databricks.yml:12:13
+
+Remove the clusters setting or set serverless to false.
+
+Name: pipeline-serverless-clusters
+Target: invalid
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/pipeline-serverless-clusters/invalid
+
+Found 1 error
+
+>>> [CLI] bundle plan -t invalid
+Error: Cannot configure clusters for a serverless pipeline
+  at resources.pipelines.my_pipeline.clusters
+  in databricks.yml:12:13
+
+Remove the clusters setting or set serverless to false.
+
+
+>>> [CLI] bundle deploy -t invalid
+Error: Cannot configure clusters for a serverless pipeline
+  at resources.pipelines.my_pipeline.clusters
+  in databricks.yml:12:13
+
+Remove the clusters setting or set serverless to false.
+
+
+=== Allow classic pipeline cluster settings
+>>> [CLI] bundle validate -t classic
+Name: pipeline-serverless-clusters
+Target: classic
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/pipeline-serverless-clusters/classic
+
+Validation OK!
+
+=== Allow serverless pipelines without cluster settings
+>>> [CLI] bundle validate -t serverless
+Name: pipeline-serverless-clusters
+Target: serverless
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/pipeline-serverless-clusters/serverless
+
+Validation OK!

acceptance/bundle/validate/pipeline_serverless_clusters/script

@@ -0,0 +1,10 @@
+title "Reject cluster settings with serverless compute before deployment"
+musterr trace $CLI bundle validate -t invalid
+musterr trace $CLI bundle plan -t invalid
+musterr trace $CLI bundle deploy -t invalid
+
+title "Allow classic pipeline cluster settings"
+trace $CLI bundle validate -t classic
+
+title "Allow serverless pipelines without cluster settings"
+trace $CLI bundle validate -t serverless

acceptance/bundle/validate/pipeline_serverless_clusters/test.toml

@@ -0,0 +1 @@
+Ignore = [".databricks"]

bundle/config/validate/fast_validate.go

@@ -29,6 +29,7 @@ func (f *fastValidate) Apply(ctx context.Context, rb *bundle.Bundle) diag.Diagno
 		// Fast mutators with only in-memory checks
 		JobClusterKeyDefined(),
 		JobTaskClusterSpec(),
+		PipelineClusterSpec(),
 
 		// Blocking mutators. Deployments will fail if these checks fail.
 		ValidateArtifactPath(),

bundle/config/validate/pipeline_cluster_spec.go

@@ -0,0 +1,41 @@
+package validate
+
+import (
+	"context"
+	"maps"
+	"slices"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/libs/diag"
+	"github.com/databricks/cli/libs/dyn"
+)
+
+// PipelineClusterSpec rejects cluster settings when serverless compute is enabled.
+func PipelineClusterSpec() bundle.ReadOnlyMutator {
+	return &pipelineClusterSpec{}
+}
+
+type pipelineClusterSpec struct{ bundle.RO }
+
+func (v *pipelineClusterSpec) Name() string {
+	return "validate:pipeline_cluster_spec"
+}
+
+func (v *pipelineClusterSpec) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics {
+	var diags diag.Diagnostics
+	for _, name := range slices.Sorted(maps.Keys(b.Config.Resources.Pipelines)) {
+		pipeline := b.Config.Resources.Pipelines[name]
+		if !pipeline.Serverless || len(pipeline.Clusters) == 0 {
+			continue
+		}
+		path := dyn.NewPath(dyn.Key("resources"), dyn.Key("pipelines"), dyn.Key(name), dyn.Key("clusters"))
+		diags = append(diags, diag.Diagnostic{
+			Severity:  diag.Error,
+			Summary:   "Cannot configure clusters for a serverless pipeline",
+			Detail:    "Remove the clusters setting or set serverless to false.",
+			Paths:     []dyn.Path{path},
+			Locations: b.Config.GetLocations(path.String()),
+		})
+	}
+	return diags
+}