Add direct-mode bundle support for AI Gateway mcp_service

.nextchanges/bundles/ai-gateway-mcp-service.md

@@ -0,0 +1 @@
+* Add bundle support for the AI Gateway `mcp_service` resource (direct engine). ([#6633](https://github.com/databricks/cli/pull/6633))

acceptance/bundle/deployment/bind/mcp_service/databricks.yml

@@ -0,0 +1,9 @@
+bundle:
+  name: test-bundle
+
+resources:
+  mcp_services:
+    mcp1:
+      parent: schemas/main.myschema
+      mcp_service_id: mysvc
+      comment: bound service

acceptance/bundle/deployment/bind/mcp_service/output.txt

@@ -0,0 +1,30 @@
+
+>>> [CLI] bundle deployment bind mcp1 main.myschema.mysvc --auto-approve
+Successfully bound mcp_service with an id 'main.myschema.mysvc'
+Run 'bundle deploy' to deploy changes to your workspace
+
+>>> [CLI] bundle summary
+Name: test-bundle
+Target: default
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default
+Resources:
+  MCP services:
+    mcp1:
+      Name: mysvc
+      URL:  [DATABRICKS_URL]/explore/data/mcp-services/main/myschema/mysvc?w=[NUMID]
+
+>>> [CLI] bundle deployment unbind mcp1
+
+>>> [CLI] bundle summary
+Name: test-bundle
+Target: default
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default
+Resources:
+  MCP services:
+    mcp1:
+      Name: mysvc
+      URL:  (not deployed)

acceptance/bundle/deployment/bind/mcp_service/script

@@ -0,0 +1,5 @@
+trace $CLI bundle deployment bind mcp1 main.myschema.mysvc --auto-approve
+trace $CLI bundle summary
+
+trace $CLI bundle deployment unbind mcp1
+trace $CLI bundle summary

acceptance/bundle/deployment/bind/mcp_service/test.toml

@@ -0,0 +1,18 @@
+# AI Gateway securables are direct-engine only.
+EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]
+
+Cloud = false
+
+Ignore = [
+    ".databricks",
+]
+
+# The bind flow issues a GET to confirm the remote resource exists before binding.
+[[Server]]
+Pattern = "GET /api/2.1/unity-catalog/mcp-services/{name}"
+Response.Body = '''
+{
+  "name": "mcp-services/main.myschema.mysvc",
+  "comment": "bound service"
+}
+'''

acceptance/bundle/invariant/configs/mcp_service.yml.tmpl

@@ -0,0 +1,9 @@
+bundle:
+  name: test-bundle-$UNIQUE_NAME
+
+resources:
+  mcp_services:
+    foo:
+      parent: schemas/main.default
+      mcp_service_id: test-mcp-service-$UNIQUE_NAME
+      comment: test mcp service

acceptance/bundle/python/mcp_services-support/databricks.yml

@@ -0,0 +1,17 @@
+bundle:
+  name: my_project
+
+sync: {paths: []} # don't need to copy files
+
+python:
+  resources:
+    - "resources:load_resources"
+  mutators:
+    - "mutators:update_mcp_service"
+
+resources:
+  mcp_services:
+    my_mcp_service_1:
+      parent: "schemas/main.default"
+      mcp_service_id: "my_mcp_service_1"
+      comment: "My MCP service"

acceptance/bundle/python/mcp_services-support/mutators.py

@@ -0,0 +1,11 @@
+from dataclasses import replace
+
+from databricks.bundles.mcp_services import McpService
+from databricks.bundles.core import mcp_service_mutator
+
+
+@mcp_service_mutator
+def update_mcp_service(mcp_service: McpService) -> McpService:
+    assert isinstance(mcp_service.comment, str)
+
+    return replace(mcp_service, comment=f"{mcp_service.comment} (updated)")

acceptance/bundle/python/mcp_services-support/output.txt

@@ -0,0 +1,28 @@
+
+>>> uv run [UV_ARGS] -q [CLI] bundle validate --output json
+{
+  "experimental": {
+    "python": {
+      "mutators": [
+        "mutators:update_mcp_service"
+      ],
+      "resources": [
+        "resources:load_resources"
+      ]
+    }
+  },
+  "resources": {
+    "mcp_services": {
+      "my_mcp_service_1": {
+        "comment": "My MCP service (updated)",
+        "mcp_service_id": "my_mcp_service_1",
+        "parent": "schemas/main.default"
+      },
+      "my_mcp_service_2": {
+        "comment": "My MCP service (2) (updated)",
+        "mcp_service_id": "my_mcp_service_2",
+        "parent": "schemas/main.default"
+      }
+    }
+  }
+}

acceptance/bundle/python/mcp_services-support/resources.py

@@ -0,0 +1,16 @@
+from databricks.bundles.core import Resources
+
+
+def load_resources() -> Resources:
+    resources = Resources()
+
+    resources.add_mcp_service(
+        "my_mcp_service_2",
+        {
+            "parent": "schemas/main.default",
+            "mcp_service_id": "my_mcp_service_2",
+            "comment": "My MCP service (2)",
+        },
+    )
+
+    return resources

acceptance/bundle/python/mcp_services-support/script

@@ -0,0 +1,5 @@
+
+trace uv run $UV_ARGS -q $CLI bundle validate --output json | \
+  jq "pick(.experimental.python, .resources)"
+
+rm -fr .databricks __pycache__

acceptance/bundle/python/mcp_services-support/test.toml

@@ -0,0 +1,4 @@
+Cloud = false # tests don't interact with APIs
+
+# mcp_services are only supported in the current version of the wheel
+EnvMatrix.PYDAB_VERSION = ["current"]

acceptance/bundle/refschema/out.fields.txt

@@ -2080,6 +2080,29 @@ resources.jobs.*.permissions[*].group_name	string	ALL
 resources.jobs.*.permissions[*].level	iam.PermissionLevel	ALL
 resources.jobs.*.permissions[*].service_principal_name	string	ALL
 resources.jobs.*.permissions[*].user_name	string	ALL
+resources.mcp_services.*.comment	string	ALL
+resources.mcp_services.*.config	*catalog.McpServiceConfig	ALL
+resources.mcp_services.*.config.include_tool_selectors	[]string	ALL
+resources.mcp_services.*.config.include_tool_selectors[*]	string	ALL
+resources.mcp_services.*.config.rate_limits	[]catalog.RateLimit	ALL
+resources.mcp_services.*.config.rate_limits[*]	catalog.RateLimit	ALL
+resources.mcp_services.*.config.rate_limits[*].key	catalog.RateLimitRateLimitKey	ALL
+resources.mcp_services.*.config.rate_limits[*].principal	string	ALL
+resources.mcp_services.*.config.rate_limits[*].renewal_period	catalog.RateLimitRateLimitRenewalPeriod	ALL
+resources.mcp_services.*.config.rate_limits[*].request_tag_key	string	ALL
+resources.mcp_services.*.config.rate_limits[*].request_tag_value	string	ALL
+resources.mcp_services.*.config.rate_limits[*].requests	int64	ALL
+resources.mcp_services.*.config.rate_limits[*].tokens	int64	ALL
+resources.mcp_services.*.config.source_connection	*catalog.McpServiceConfigSourceConnection	ALL
+resources.mcp_services.*.config.source_connection.is_deleted	bool	ALL
+resources.mcp_services.*.config.source_connection.name	string	ALL
+resources.mcp_services.*.id	string	INPUT
+resources.mcp_services.*.lifecycle	resources.Lifecycle	INPUT
+resources.mcp_services.*.lifecycle.prevent_destroy	bool	INPUT
+resources.mcp_services.*.mcp_service_id	string	ALL
+resources.mcp_services.*.modified_status	string	INPUT
+resources.mcp_services.*.parent	string	ALL
+resources.mcp_services.*.url	string	INPUT
 resources.model_services.*.comment	string	ALL
 resources.model_services.*.config	*catalog.ModelServiceConfig	ALL
 resources.model_services.*.config.inference_table	*catalog.InferenceTableConfig	ALL

acceptance/bundle/resources/mcp_services/basic/databricks.yml

@@ -0,0 +1,9 @@
+bundle:
+  name: test-bundle
+
+resources:
+  mcp_services:
+    mcp1:
+      parent: schemas/main.myschema
+      mcp_service_id: myservice
+      comment: COMMENT1

acceptance/bundle/resources/mcp_services/basic/output.txt

@@ -0,0 +1,92 @@
+
+=== Initial summary before deploy
+>>> [CLI] bundle summary -o json
+{
+  "comment": "COMMENT1",
+  "mcp_service_id": "myservice",
+  "modified_status": "created",
+  "parent": "schemas/main.myschema"
+}
+
+=== Verify it does not exist yet
+>>> musterr [CLI] ai-gateway get-mcp-service mcp-services/main.myschema.myservice
+Error: Resource catalog.McpService not found: main.myschema.myservice
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
+Created mcp_services.mcp1
+Files: 3 uploaded, 0 deleted
+Resources: 1 created, 0 changed, 0 deleted, 0 unchanged
+
+>>> print_requests.py //unity
+{
+  "method": "POST",
+  "path": "/api/2.1/unity-catalog/mcp-services",
+  "q": {
+    "mcp_service_id": "myservice",
+    "parent": "schemas/main.myschema"
+  },
+  "body": {
+    "comment": "COMMENT1"
+  }
+}
+
+=== Summary should show the id and the Catalog Explorer url
+>>> [CLI] bundle summary -o json
+{
+  "id": "main.myschema.myservice",
+  "url": "[DATABRICKS_URL]/explore/data/mcp-services/main/myschema/myservice?w=[NUMID]"
+}
+
+=== Verify deployment
+>>> [CLI] ai-gateway get-mcp-service mcp-services/main.myschema.myservice
+{
+  "name": "mcp-services/main.myschema.myservice",
+  "comment": "COMMENT1"
+}
+
+=== Update comment (should update in place, not recreate)
+>>> update_file.py databricks.yml COMMENT1 COMMENT2
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
+Updated mcp_services.mcp1
+Files: 1 uploaded, 0 deleted
+Resources: 0 created, 1 changed, 0 deleted, 0 unchanged
+
+>>> print_requests.py //unity
+{
+  "method": "PATCH",
+  "path": "/api/2.1/unity-catalog/mcp-services/main.myschema.myservice",
+  "q": {
+    "update_mask": "*"
+  },
+  "body": {
+    "comment": "COMMENT2"
+  }
+}
+
+>>> [CLI] ai-gateway get-mcp-service mcp-services/main.myschema.myservice
+"COMMENT2"
+
+=== Change an immutable field (should plan a recreate)
+>>> update_file.py databricks.yml myservice myservice-renamed
+
+>>> [CLI] bundle plan
+recreate mcp_services.mcp1
+
+Plan: 1 to add, 0 to change, 1 to delete, 0 unchanged
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.mcp_services.mcp1
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default
+
+Destroy: 1 deleted
+
+>>> print_requests.py //unity
+{
+  "method": "DELETE",
+  "path": "/api/2.1/unity-catalog/mcp-services/main.myschema.myservice"
+}

acceptance/bundle/resources/mcp_services/basic/script

@@ -0,0 +1,29 @@
+title "Initial summary before deploy"
+trace $CLI bundle summary -o json | jq .resources.mcp_services.mcp1
+
+title "Verify it does not exist yet"
+trace musterr $CLI ai-gateway get-mcp-service mcp-services/main.myschema.myservice
+
+trace $CLI bundle deploy
+trace print_requests.py //unity
+
+title "Summary should show the id and the Catalog Explorer url"
+trace $CLI bundle summary -o json | jq ".resources.mcp_services.mcp1 | {id, url}"
+
+title "Verify deployment"
+trace $CLI ai-gateway get-mcp-service mcp-services/main.myschema.myservice | jq '{name, comment}'
+
+title "Update comment (should update in place, not recreate)"
+trace update_file.py databricks.yml COMMENT1 COMMENT2
+trace $CLI bundle deploy
+trace print_requests.py //unity
+trace $CLI ai-gateway get-mcp-service mcp-services/main.myschema.myservice | jq .comment
+
+title "Change an immutable field (should plan a recreate)"
+trace update_file.py databricks.yml myservice myservice-renamed
+trace $CLI bundle plan
+
+trace $CLI bundle destroy --auto-approve
+trace print_requests.py //unity
+
+rm -f out.requests.txt

acceptance/bundle/resources/mcp_services/basic/test.toml

@@ -0,0 +1,7 @@
+# Local only: this test inspects the recorded request stream (print_requests),
+# which has no equivalent against a real workspace.
+Cloud = false
+
+Ignore = [
+    ".databricks",
+]

acceptance/bundle/resources/mcp_services/lifecycle/databricks.yml.tmpl

@@ -0,0 +1,12 @@
+bundle:
+  name: deploy-mcp-service-test-$UNIQUE_NAME
+
+resources:
+  mcp_services:
+    mcp:
+      parent: schemas/main.default
+      mcp_service_id: test_mcp_$UNIQUE_NAME
+      comment: "Points at a UC connection hosting an MCP server"
+      config:
+        source_connection:
+          name: connections/mcp_conn_$UNIQUE_NAME

acceptance/bundle/resources/mcp_services/lifecycle/output.txt

@@ -0,0 +1,23 @@
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-mcp-service-test-[UNIQUE_NAME]/default/files...
+Created mcp_services.mcp
+Files: 4 uploaded, 0 deleted
+Resources: 1 created, 0 changed, 0 deleted, 0 unchanged
+
+>>> [CLI] ai-gateway get-mcp-service mcp-services/main.default.test_mcp_[UNIQUE_NAME]
+{
+  "name": "mcp-services/main.default.test_mcp_[UNIQUE_NAME]",
+  "comment": "Points at a UC connection hosting an MCP server"
+}
+
+>>> [CLI] bundle plan
+Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.mcp_services.mcp
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/deploy-mcp-service-test-[UNIQUE_NAME]/default
+
+Destroy: 1 deleted