Skip to content

direct: Fix jobs and pipelines RemoteType to be superset of StateType #4443

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
denik wants to merge 17 commits into
base: main
Choose a base branch
from
Open

direct: Fix jobs and pipelines RemoteType to be superset of StateType #4443

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
367e222
direct: Fix jobs and pipelines RemoteType to be superset of StateType
denik Feb 4, 2026
12be2c6
ignore_remote_changes for pipeline.id
denik Feb 4, 2026
e22809d
update tests
denik Feb 4, 2026
6abb0b9
update test missing-map-key
denik Feb 4, 2026
c5d5599
update test
denik Feb 5, 2026
cd6aa80
update migrate/basic test
denik Feb 5, 2026
b204e45
rm unnecessary requests
denik Feb 5, 2026
c85e127
update refschema
denik Feb 5, 2026
f0b8c88
fix
denik Feb 5, 2026
6ceda96
update
denik Feb 6, 2026
75d3e6d
Add tests verifying JobRemote and PipelineRemote cover all SDK fields
denik Feb 6, 2026
3ffbe79
Drop "id" changes from pipeline
denik Feb 6, 2026
00de2be
Merge branch 'main' into denik/jobs-pipelines-remotetype
denik Feb 6, 2026
6299e60
update test output
denik Feb 6, 2026
0869f66
update default-python
denik Feb 6, 2026
7ddc329
update migrate/default-python
denik Feb 6, 2026
17aa0f1
update tests
denik Feb 6, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
128 changes: 63 additions & 65 deletions acceptance/bundle/artifacts/whl_dynamic/out.plan_update.direct.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,77 +77,75 @@
"remote_state": {
"created_time": [UNIX_TIME_MILLIS],
"creator_user_name": "[USERNAME]",
"deployment": {
"kind": "BUNDLE",
"metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json"
},
"edit_mode": "UI_LOCKED",
"email_notifications": {},
"environments": [
{
"environment_key": "test_env",
"spec": {
"client": "1",
"dependencies": [
"/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][2]-py3-none-any.whl"
]
}
}
],
"format": "MULTI_TASK",
"job_id": [NUMID],
"max_concurrent_runs": 1,
"name": "[default] My Wheel Job",
"queue": {
"enabled": true
},
"run_as_user_name": "[USERNAME]",
"settings": {
"deployment": {
"kind": "BUNDLE",
"metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json"
"tasks": [
{
"environment_key": "test_env",
"python_wheel_task": {
"entry_point": "run",
"package_name": "my_test_code"
},
"task_key": "ServerlessTestTask"
},
"edit_mode": "UI_LOCKED",
"email_notifications": {},
"environments": [
{
"environment_key": "test_env",
"spec": {
"client": "1",
"dependencies": [
"/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][2]-py3-none-any.whl"
]
{
"existing_cluster_id": "0717-132531-5opeqon1",
"for_each_task": {
"inputs": "[1]",
"task": {
"existing_cluster_id": "0717-132531-5opeqon1",
"libraries": [
{
"whl": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][2]-py3-none-any.whl"
}
],
"python_wheel_task": {
"entry_point": "run",
"package_name": "my_test_code"
},
"task_key": "SubTask"
}
}
],
"format": "MULTI_TASK",
"max_concurrent_runs": 1,
"name": "[default] My Wheel Job",
"queue": {
"enabled": true
},
"tasks": [
{
"environment_key": "test_env",
"python_wheel_task": {
"entry_point": "run",
"package_name": "my_test_code"
},
"task_key": "ServerlessTestTask"
},
{
"existing_cluster_id": "0717-132531-5opeqon1",
"for_each_task": {
"inputs": "[1]",
"task": {
"existing_cluster_id": "0717-132531-5opeqon1",
"libraries": [
{
"whl": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][2]-py3-none-any.whl"
}
],
"python_wheel_task": {
"entry_point": "run",
"package_name": "my_test_code"
},
"task_key": "SubTask"
}
},
"libraries": [
{
"whl": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][2]-py3-none-any.whl"
},
{
"whl": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/other_test_code-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl"
}
],
"python_wheel_task": {
"entry_point": "run",
"package_name": "my_test_code"
"libraries": [
{
"whl": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][2]-py3-none-any.whl"
},
"task_key": "TestTask"
}
],
"timeout_seconds": 0,
"webhook_notifications": {}
}
{
"whl": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/other_test_code-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl"
}
],
"python_wheel_task": {
"entry_point": "run",
"package_name": "my_test_code"
},
"task_key": "TestTask"
}
],
"timeout_seconds": 0,
"webhook_notifications": {}
},
"changes": {
"email_notifications": {
Expand Down
2 changes: 1 addition & 1 deletion acceptance/bundle/bundle_tag/url_ref/out.deploy.direct.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
Error: cannot plan resources.jobs.bar: cannot resolve "${resources.jobs.foo.url}": schema mismatch: url: field "url" not found in jobs.JobSettings; url: field "url" not found in jobs.Job
Error: cannot plan resources.jobs.bar: cannot resolve "${resources.jobs.foo.url}": schema mismatch: url: field "url" not found in jobs.JobSettings; url: field "url" not found in dresources.JobRemote

Error: planning failed

Expand Down
2 changes: 1 addition & 1 deletion acceptance/bundle/bundle_tag/url_ref/out.plan.direct.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
Error: cannot plan resources.jobs.bar: cannot resolve "${resources.jobs.foo.url}": schema mismatch: url: field "url" not found in jobs.JobSettings; url: field "url" not found in jobs.Job
Error: cannot plan resources.jobs.bar: cannot resolve "${resources.jobs.foo.url}": schema mismatch: url: field "url" not found in jobs.JobSettings; url: field "url" not found in dresources.JobRemote

Error: planning failed

Expand Down
30 changes: 14 additions & 16 deletions acceptance/bundle/deploy/readplan/basic/out.plan_skip.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,24 +9,22 @@
"remote_state": {
"created_time": [UNIX_TIME_MILLIS],
"creator_user_name": "[USERNAME]",
"deployment": {
"kind": "BUNDLE",
"metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json"
},
"edit_mode": "UI_LOCKED",
"email_notifications": {},
"format": "MULTI_TASK",
"job_id": [NUMID],
"max_concurrent_runs": 1,
"name": "test-job",
"queue": {
"enabled": true
},
"run_as_user_name": "[USERNAME]",
"settings": {
"deployment": {
"kind": "BUNDLE",
"metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json"
},
"edit_mode": "UI_LOCKED",
"email_notifications": {},
"format": "MULTI_TASK",
"max_concurrent_runs": 1,
"name": "test-job",
"queue": {
"enabled": true
},
"timeout_seconds": 0,
"webhook_notifications": {}
}
"timeout_seconds": 0,
"webhook_notifications": {}
},
"changes": {
"email_notifications": {
Expand Down
2 changes: 1 addition & 1 deletion acceptance/bundle/migrate/basic/databricks.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ resources:
myjob_name: ${resources.jobs.test_job.name}
volume_catalog_name: ${resources.volumes.test_volume.catalog_name}

# remote field, int, null
# remote field, int
myjob_timeout: ${resources.jobs.test_job.timeout_seconds}

# remote field, string:
Expand Down
2 changes: 1 addition & 1 deletion acceptance/bundle/migrate/basic/out.new_state.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@
"tags": {
"myjob_id": "[NUMID]",
"myjob_name": "Test Migration Job",
"myjob_timeout": "",
"myjob_timeout": "0",
Copy link
Contributor

@pietern pietern Feb 6, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Intentional?

Copy link
Contributor

@pietern pietern Feb 6, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah, I see this wasn't resolved before, but now it is.

"myvolume_id": "mycat.myschema.myvol",
"volume_catalog_name": "mycat",
"volume_storage_location": "s3://deco-uc-prod-isolated-aws-us-east-1/metastore/[UUID]/volumes/[UUID]"
Expand Down
110 changes: 57 additions & 53 deletions acceptance/bundle/migrate/basic/out.plan_update.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,33 +32,31 @@
"remote_state": {
"created_time": [UNIX_TIME_MILLIS][0],
"creator_user_name": "[USERNAME]",
"deployment": {
"kind": "BUNDLE",
"metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/migrate-basic-test/dev/state/metadata.json"
},
"edit_mode": "UI_LOCKED",
"email_notifications": {},
"format": "MULTI_TASK",
"job_id": [NUMID],
"max_concurrent_runs": 1,
"name": "Test Migration Job",
"queue": {
"enabled": true
},
"run_as_user_name": "[USERNAME]",
"settings": {
"deployment": {
"kind": "BUNDLE",
"metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/migrate-basic-test/dev/state/metadata.json"
},
"edit_mode": "UI_LOCKED",
"email_notifications": {},
"format": "MULTI_TASK",
"max_concurrent_runs": 1,
"name": "Test Migration Job",
"queue": {
"enabled": true
},
"tasks": [
{
"notebook_task": {
"notebook_path": "/Workspace/Users/[USERNAME]/.bundle/migrate-basic-test/dev/files/notebook",
"source": "WORKSPACE"
},
"task_key": "main"
}
],
"timeout_seconds": 0,
"webhook_notifications": {}
}
"tasks": [
{
"notebook_task": {
"notebook_path": "/Workspace/Users/[USERNAME]/.bundle/migrate-basic-test/dev/files/notebook",
"source": "WORKSPACE"
},
"task_key": "main"
}
],
"timeout_seconds": 0,
"webhook_notifications": {}
},
"changes": {
"email_notifications": {
Expand Down Expand Up @@ -136,46 +134,46 @@
"tags": {
"myjob_id": "[NUMID]",
"myjob_name": "Test Migrated Job",
"myjob_timeout": "",
"myjob_timeout": "${resources.jobs.test_job.timeout_seconds}",
"myvolume_id": "mycat.myschema.myvol",
"volume_catalog_name": "mycat",
"volume_storage_location": "s3://deco-uc-prod-isolated-aws-us-east-1/metastore/[UUID]/volumes/[UUID]"
}
},
"vars": {
"tags.myjob_timeout": "${resources.jobs.test_job.timeout_seconds}"
}
},
"remote_state": {
"channel": "CURRENT",
"creator_user_name": "[USERNAME]",
"deployment": {
"kind": "BUNDLE",
"metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/migrate-basic-test/dev/state/metadata.json"
},
"edition": "ADVANCED",
"id": "[UUID]",
"last_modified": [UNIX_TIME_MILLIS][1],
"libraries": [
{
"notebook": {
"path": "/Workspace/Users/[USERNAME]/.bundle/migrate-basic-test/dev/files/pipeline"
}
}
],
"name": "Test Migration Pipeline",
"pipeline_id": "[UUID]",
"run_as_user_name": "[USERNAME]",
"spec": {
"channel": "CURRENT",
"deployment": {
"kind": "BUNDLE",
"metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/migrate-basic-test/dev/state/metadata.json"
},
"edition": "ADVANCED",
"id": "[UUID]",
"libraries": [
{
"notebook": {
"path": "/Workspace/Users/[USERNAME]/.bundle/migrate-basic-test/dev/files/pipeline"
}
}
],
"name": "Test Migration Pipeline",
"storage": "dbfs:/pipelines/[UUID]",
"tags": {
"myjob_id": "[NUMID]",
"myjob_name": "Test Migration Job",
"myjob_timeout": "",
"myvolume_id": "mycat.myschema.myvol",
"volume_catalog_name": "mycat",
"volume_storage_location": "s3://deco-uc-prod-isolated-aws-us-east-1/metastore/[UUID]/volumes/[UUID]"
}
},
"state": "IDLE"
"state": "IDLE",
"storage": "dbfs:/pipelines/[UUID]",
"tags": {
"myjob_id": "[NUMID]",
"myjob_name": "Test Migration Job",
"myjob_timeout": "0",
"myvolume_id": "mycat.myschema.myvol",
"volume_catalog_name": "mycat",
"volume_storage_location": "s3://deco-uc-prod-isolated-aws-us-east-1/metastore/[UUID]/volumes/[UUID]"
}
},
"changes": {
"name": {
Expand All @@ -194,6 +192,12 @@
"old": "Test Migration Job",
"new": "Test Migrated Job",
"remote": "Test Migration Job"
},
"tags['myjob_timeout']": {
"action": "update",
"old": "0",
"new": "${resources.jobs.test_job.timeout_seconds}",
"remote": "0"
Copy link
Contributor

@pietern pietern Feb 6, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Will this downgrade during deploy?

Copy link
Contributor Author

@denik denik Feb 6, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We don't do any downgrades (yet).

}
}
},
Expand Down
8 changes: 2 additions & 6 deletions acceptance/bundle/migrate/basic/output.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,14 +49,10 @@ Error: already using direct engine
Details: [TEST_TMP_DIR]/.databricks/bundle/dev/resources.json: local direct state serial=6 lineage="[UUID]"

>>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle plan
update pipelines.test_pipeline

Plan: 0 to add, 1 to change, 0 to delete, 2 unchanged
Plan: 0 to add, 0 to change, 0 to delete, 3 unchanged

>>> DATABRICKS_BUNDLE_ENGINE= [CLI] bundle plan
update pipelines.test_pipeline

Plan: 0 to add, 1 to change, 0 to delete, 2 unchanged
Plan: 0 to add, 0 to change, 0 to delete, 3 unchanged

>>> print_requests.py --get //jobs/get
{
Expand Down
4 changes: 2 additions & 2 deletions acceptance/bundle/migrate/basic/script
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,8 @@ trace musterr $CLI bundle deployment migrate

rm out.requests.txt
# Both DATABRICKS_BUNDLE_ENGINE=direct and DATABRICKS_BUNDLE_ENGINE= work
trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle plan | contains.py "2 unchanged" # should be "3 unchanged" once permanent drift is fixed
trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle plan | contains.py "2 unchanged"
trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle plan | contains.py "3 unchanged"
trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle plan | contains.py "3 unchanged"
trace print_requests.py --get //jobs/get | contains.py 'engine/direct'

trace DATABRICKS_BUNDLE_ENGINE="" $CLI bundle deploy
Expand Down
Loading
Loading