From 83de54779f5613d4956d8ec07fcffecf8c388142 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 26 Oct 2026 05:35:24 +0000 Subject: [PATCH 1/2] Default deployment history to enforced file-backed recording --- acceptance/bin/print_state.py | 17 +- .../readplan/lineage-mismatch/plan.dms.json | 2 +- .../databricks.yml.tmpl | 3 - .../dms/bind-unbind-unsupported/output.txt | 10 +- .../bundle/dms/bind-unbind-unsupported/script | 7 +- .../bundle/dms/declined-deploy/output.txt | 30 +-- acceptance/bundle/dms/depends-on/output.txt | 52 ++++- .../dms/deployment-metadata-change/output.txt | 6 +- acceptance/bundle/dms/empty-grants/output.txt | 14 +- .../bundle/dms/existing-state/output.txt | 2 +- acceptance/bundle/dms/existing-state/script | 2 +- .../bundle/dms/failed-delete/output.txt | 54 ++++- acceptance/bundle/dms/failed-delete/script | 4 +- .../bundle/dms/failed-recreate/output.txt | 19 +- acceptance/bundle/dms/failed-recreate/script | 2 +- .../dms/failed-update-permissions/output.txt | 92 +++++++- .../dms/mismatch-before-read/output.txt | 12 +- .../bundle/dms/mismatch-before-read/script | 7 +- .../bundle/dms/multiple-resources/output.txt | 198 +++++++++++++++++- acceptance/bundle/dms/no-drift/output.txt | 101 ++++++++- acceptance/bundle/dms/no-drift/script | 1 + acceptance/bundle/dms/no-resources/output.txt | 24 +-- .../operation-upload-fails-once/output.txt | 61 ++++-- .../dms/operation-upload-fails-once/script | 9 +- .../bundle/dms/plan-without-dms/output.txt | 2 +- acceptance/bundle/dms/plan-without-dms/script | 6 +- acceptance/bundle/dms/provenance/output.txt | 28 ++- .../bundle/dms/record-failure/output.txt | 6 +- acceptance/bundle/dms/record-failure/script | 2 +- .../dms/stale-local-state/databricks.yml.tmpl | 3 - .../bundle/dms/stale-local-state/output.txt | 28 ++- .../bundle/dms/stale-local-state/script | 22 +- acceptance/bundle/dms/stale-plan/output.txt | 25 ++- .../bundle/dms/state-features/output.txt | 53 +++-- acceptance/bundle/dms/state-features/script | 17 +- .../bundle/dms/successful-recreate/output.txt | 30 ++- .../dms/version-never-created/output.txt | 45 ++-- .../bundle/dms/version-never-created/script | 9 +- bundle/config/experimental.go | 5 +- bundle/direct/apply.go | 2 +- bundle/direct/dstate/state.go | 34 ++- bundle/direct/dstate/state_test.go | 42 +++- bundle/phases/bind.go | 7 +- bundle/phases/deploy.go | 34 +-- bundle/phases/destroy.go | 53 +++-- bundle/phases/dms.go | 80 +++++-- bundle/phases/telemetry.go | 2 +- bundle/statemgmt/state_pull.go | 6 + cmd/bundle/utils/destroy.go | 6 +- cmd/bundle/utils/process.go | 95 +++++++-- cmd/bundle/utils/process_test.go | 20 -- 51 files changed, 1048 insertions(+), 343 deletions(-) diff --git a/acceptance/bin/print_state.py b/acceptance/bin/print_state.py index 6ad674342c3..1f60d0b27b4 100755 --- a/acceptance/bin/print_state.py +++ b/acceptance/bin/print_state.py @@ -174,23 +174,26 @@ def get_last_version_id(target): def print_recorded_state(filename, target): - """Print the state file with its resources filled in from the deployment metadata service. + """Print DMS resources using the state file's header. - While recording, the file itself carries only the header - the service holds the resources - so - printing it raw would show an empty state and differ from the same test's non-recording run. + This checks history uploads even though resources.json remains authoritative. + Use --no-dms to inspect the file state instead. """ data = json.loads(open(filename).read()) # Recording stamps each resource payload with the deployment and version; drop it here so the # printed state matches a non-recording run without every caller piping through nostamp. data["state"] = scrub(get_recorded_state(target)) - # The file's serial is a copy of the DMS version to ensure outdated resource state is not read. - # Rebuild in header order with the authoritative serial from the deployment. + # Pair DMS resources with the DMS version. The file serial can advance independently. last_version_id = get_last_version_id(target) serial = int(last_version_id) if last_version_id else 0 # Recording itself is not what these tests assert, so drop the feature that marks it. - features = {k: v for k, v in (data.get("features") or {}).items() if k != "deployment_history"} + features = { + k: v + for k, v in (data.get("features") or {}).items() + if k not in {"deployment_history", "deployment_history_best_effort"} + } rebuilt = {} for key in ("state_version", "cli_version", "lineage"): @@ -213,7 +216,7 @@ def main(): "--no-dms", action="store_true", help="Print resources.json as-is without fetching state from DMS. " - "Use in bundle/dms tests to assert the tombstone content of the file.", + "Use in bundle/dms tests to assert the file state and its feature markers.", ) args = parser.parse_args() diff --git a/acceptance/bundle/deploy/readplan/lineage-mismatch/plan.dms.json b/acceptance/bundle/deploy/readplan/lineage-mismatch/plan.dms.json index 58e072b84f1..77503001c1c 100644 --- a/acceptance/bundle/deploy/readplan/lineage-mismatch/plan.dms.json +++ b/acceptance/bundle/deploy/readplan/lineage-mismatch/plan.dms.json @@ -5,6 +5,6 @@ "serial": 0, "plan": {}, "features": { - "deployment_history": {} + "deployment_history_best_effort": {} } } diff --git a/acceptance/bundle/dms/bind-unbind-unsupported/databricks.yml.tmpl b/acceptance/bundle/dms/bind-unbind-unsupported/databricks.yml.tmpl index e60f4605610..92a70710626 100644 --- a/acceptance/bundle/dms/bind-unbind-unsupported/databricks.yml.tmpl +++ b/acceptance/bundle/dms/bind-unbind-unsupported/databricks.yml.tmpl @@ -1,8 +1,5 @@ bundle: name: dms-bind-unbind-$UNIQUE_NAME -experimental: - deployment_history: true - resources: jobs: foo: diff --git a/acceptance/bundle/dms/bind-unbind-unsupported/output.txt b/acceptance/bundle/dms/bind-unbind-unsupported/output.txt index d2d0a366778..e2781fef6fb 100644 --- a/acceptance/bundle/dms/bind-unbind-unsupported/output.txt +++ b/acceptance/bundle/dms/bind-unbind-unsupported/output.txt @@ -4,13 +4,17 @@ Error: bind is not supported for a bundle target that records deployment history -=== Deploy so the bundle records state, then unbind is refused too +=== Unbind is allowed for file-backed history state >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-bind-unbind-[UNIQUE_NAME]/default/files... Created jobs.foo Files: 2 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ->>> musterr [CLI] bundle deployment unbind foo -Error: unbind is not supported for a bundle target that records deployment history +>>> [CLI] bundle deployment unbind foo +>>> jq {features, state} .databricks/bundle/default/resources.json +{ + "features": null, + "state": {} +} diff --git a/acceptance/bundle/dms/bind-unbind-unsupported/script b/acceptance/bundle/dms/bind-unbind-unsupported/script index 03d9cf34a31..7437d908eb2 100644 --- a/acceptance/bundle/dms/bind-unbind-unsupported/script +++ b/acceptance/bundle/dms/bind-unbind-unsupported/script @@ -8,7 +8,8 @@ job_id=$($CLI jobs create --json '{"name": "bind-target-'$UNIQUE_NAME'"}' | jq - add_repl "$job_id" JOB_ID trace musterr $CLI bundle deployment bind foo "$job_id" -# unbind refuses when it opens an existing recording state, so it needs a deployment first. -title "Deploy so the bundle records state, then unbind is refused too" +# File state remains writable even when it carries the history marker. +title "Unbind is allowed for file-backed history state" trace $CLI bundle deploy -trace musterr $CLI bundle deployment unbind foo +trace $CLI bundle deployment unbind foo +trace jq '{features, state}' .databricks/bundle/default/resources.json diff --git a/acceptance/bundle/dms/declined-deploy/output.txt b/acceptance/bundle/dms/declined-deploy/output.txt index 3ef52219df5..b3e2931c756 100644 --- a/acceptance/bundle/dms/declined-deploy/output.txt +++ b/acceptance/bundle/dms/declined-deploy/output.txt @@ -67,7 +67,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "lineage": "[UUID]", "serial": 1, "features": { - "deployment_history": {} + "deployment_history_best_effort": {} } } @@ -79,18 +79,10 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "method": "GET", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]" } -{ - "method": "GET", - "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/resources" -} { "method": "GET", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]" } -{ - "method": "GET", - "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/resources" -} >>> print_state.py --no-dms { @@ -99,9 +91,17 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "lineage": "[UUID]", "serial": 1, "features": { - "deployment_history": {} + "deployment_history_best_effort": {} }, - "state": {} + "state": { + "resources.schemas.foo": { + "__id__": "main.dms_declined_deploy_schema_[UNIQUE_NAME]", + "state": { + "catalog_name": "main", + "name": "dms_declined_deploy_schema_[UNIQUE_NAME]" + } + } + } } === A destructive change without --auto-approve is declined: this console cannot prompt @@ -124,10 +124,6 @@ Files: 2 uploaded, 0 deleted "method": "GET", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]" } -{ - "method": "GET", - "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/resources" -} === The recorded resources are unchanged by the decline: the schema is still at version 1 >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[DEPLOYMENT_ID] -o json @@ -163,10 +159,6 @@ false "method": "GET", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]" } -{ - "method": "GET", - "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/resources" -} { "method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions", diff --git a/acceptance/bundle/dms/depends-on/output.txt b/acceptance/bundle/dms/depends-on/output.txt index c1b9beb1dec..fce9158905f 100644 --- a/acceptance/bundle/dms/depends-on/output.txt +++ b/acceptance/bundle/dms/depends-on/output.txt @@ -5,7 +5,7 @@ "plan_version": [PLAN_VERSION], "cli_version": "[CLI_VERSION]", "features": { - "deployment_history": {} + "deployment_history_best_effort": {} } } @@ -61,7 +61,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "lineage": "[UUID]", "serial": 1, "features": { - "deployment_history": {} + "deployment_history_best_effort": {} } } @@ -91,9 +91,53 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged "lineage": "[UUID]", "serial": 1, "features": { - "deployment_history": {} + "deployment_history_best_effort": {} }, - "state": {} + "state": { + "resources.jobs.child": { + "__id__": "[CHILD_ID]", + "state": { + "deployment": { + "deployment_id": "[DEPLOYMENT_ID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-depends-on-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "description": "depends on [PARENT_ID]", + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "child", + "queue": { + "enabled": true + } + }, + "depends_on": [ + { + "node": "resources.jobs.parent", + "label": "${resources.jobs.parent.id}" + } + ] + }, + "resources.jobs.parent": { + "__id__": "[PARENT_ID]", + "state": { + "deployment": { + "deployment_id": "[DEPLOYMENT_ID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-depends-on-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "parent", + "queue": { + "enabled": true + } + } + } + } } >>> [CLI] bundle destroy --auto-approve diff --git a/acceptance/bundle/dms/deployment-metadata-change/output.txt b/acceptance/bundle/dms/deployment-metadata-change/output.txt index 675d23ca70d..ee12a197034 100644 --- a/acceptance/bundle/dms/deployment-metadata-change/output.txt +++ b/acceptance/bundle/dms/deployment-metadata-change/output.txt @@ -67,7 +67,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "lineage": "[UUID]", "serial": 1, "features": { - "deployment_history": {} + "deployment_history_best_effort": {} } } @@ -126,9 +126,9 @@ A common practice is to use a username or principal name in this path, i.e. use "plan_version": [PLAN_VERSION], "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", - "serial": 1, + "serial": 2, "features": { - "deployment_history": {} + "deployment_history_best_effort": {} } } diff --git a/acceptance/bundle/dms/empty-grants/output.txt b/acceptance/bundle/dms/empty-grants/output.txt index 0a6cbabcbce..ed3836a9c80 100644 --- a/acceptance/bundle/dms/empty-grants/output.txt +++ b/acceptance/bundle/dms/empty-grants/output.txt @@ -138,7 +138,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "lineage": "[UUID]", "serial": 2, "features": { - "deployment_history": {} + "deployment_history_best_effort": {} } } @@ -163,9 +163,17 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "lineage": "[UUID]", "serial": 2, "features": { - "deployment_history": {} + "deployment_history_best_effort": {} }, - "state": {} + "state": { + "resources.schemas.foo": { + "__id__": "main.dms_empty_grants_[UNIQUE_NAME]", + "state": { + "catalog_name": "main", + "name": "dms_empty_grants_[UNIQUE_NAME]" + } + } + } } >>> [CLI] bundle destroy --auto-approve diff --git a/acceptance/bundle/dms/existing-state/output.txt b/acceptance/bundle/dms/existing-state/output.txt index 4057525509b..841ffabe13b 100644 --- a/acceptance/bundle/dms/existing-state/output.txt +++ b/acceptance/bundle/dms/existing-state/output.txt @@ -73,6 +73,6 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged { "state_version": 3, "features": { - "deployment_history": {} + "deployment_history_best_effort": {} } } diff --git a/acceptance/bundle/dms/existing-state/script b/acceptance/bundle/dms/existing-state/script index e64ca6febe6..837b8bc1d8f 100644 --- a/acceptance/bundle/dms/existing-state/script +++ b/acceptance/bundle/dms/existing-state/script @@ -16,7 +16,7 @@ musterr trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle deploy title "A leftover local WAL with the wrong lineage is still refused" mkdir -p .databricks/bundle/default -printf '{"lineage":"stale","serial":99,"state_version":2,"features":{"deployment_history":{}}}\n' > .databricks/bundle/default/resources.json.wal +printf '{"lineage":"stale","serial":99,"state_version":2,"features":{"deployment_history_best_effort":{}}}\n' > .databricks/bundle/default/resources.json.wal musterr trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle deploy title "Destroy clears the deployment. With nothing recorded remotely the bundle is treated as new, so recording can be enabled once the stale WAL is gone" diff --git a/acceptance/bundle/dms/failed-delete/output.txt b/acceptance/bundle/dms/failed-delete/output.txt index e3241b3d66a..db348b73d61 100644 --- a/acceptance/bundle/dms/failed-delete/output.txt +++ b/acceptance/bundle/dms/failed-delete/output.txt @@ -67,7 +67,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "lineage": "[UUID]", "serial": 1, "features": { - "deployment_history": {} + "deployment_history_best_effort": {} } } @@ -92,9 +92,28 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "lineage": "[UUID]", "serial": 1, "features": { - "deployment_history": {} + "deployment_history_best_effort": {} }, - "state": {} + "state": { + "resources.jobs.stuck": { + "__id__": "[STUCK_ID]", + "state": { + "deployment": { + "deployment_id": "[DEPLOYMENT_ID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-failed-delete-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "stuck", + "queue": { + "enabled": true + } + } + } + } } === A resource that fails to delete is recorded as a failed operation carrying the error, so the history says why rather than showing the resource gone @@ -176,7 +195,7 @@ API message: Fault injected by test. } ] -=== The local state file records failed version 2 +=== The local state file keeps the resource and its serial after the failed delete >>> print_state.py --no-dms { "state_version": 3, @@ -184,14 +203,33 @@ API message: Fault injected by test. "lineage": "[UUID]", "serial": 2, "features": { - "deployment_history": {} + "deployment_history_best_effort": {} }, - "state": {} + "state": { + "resources.jobs.stuck": { + "__id__": "[STUCK_ID]", + "state": { + "deployment": { + "deployment_id": "[DEPLOYMENT_ID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-failed-delete-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "stuck", + "queue": { + "enabled": true + } + } + } + } } === The remote checkpoint stays at serial 1 while DMS advances to version 2 >>> MSYS_NO_PATHCONV=1 [CLI] workspace export /Workspace/Users/[USERNAME]/.bundle/dms-failed-delete-[UNIQUE_NAME]/default/state/resources.json --format RAW {"serial":1} ->>> print_state.py -{"serial":2} +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments get-deployment deployments/[DEPLOYMENT_ID] -o json +{"last_version_id":"2"} diff --git a/acceptance/bundle/dms/failed-delete/script b/acceptance/bundle/dms/failed-delete/script index 2b733207f0a..5552fd18bad 100644 --- a/acceptance/bundle/dms/failed-delete/script +++ b/acceptance/bundle/dms/failed-delete/script @@ -29,12 +29,12 @@ trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${d title "The failed operation records the error message" trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-operations "deployments/${deployment_id}/versions/2" -o json | jq -S 'map({resource_key, action_type, status, error_message})' -title "The local state file records failed version 2" +title "The local state file keeps the resource and its serial after the failed delete" trace print_state.py --no-dms title "The remote checkpoint stays at serial 1 while DMS advances to version 2" trace MSYS_NO_PATHCONV=1 $CLI workspace export "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-failed-delete-${UNIQUE_NAME}/default/state/resources.json" --format RAW | jq -c '{serial}' -trace print_state.py | jq -c '{serial}' +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments get-deployment "deployments/${deployment_id}" -o json | jq -c '{last_version_id}' # Drain any accumulated requests (plan, state reads) so the harness has nothing to diff. rm -f "$OUT_REQUESTS" diff --git a/acceptance/bundle/dms/failed-recreate/output.txt b/acceptance/bundle/dms/failed-recreate/output.txt index 860460c3375..f2cb631690a 100644 --- a/acceptance/bundle/dms/failed-recreate/output.txt +++ b/acceptance/bundle/dms/failed-recreate/output.txt @@ -67,7 +67,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "lineage": "[UUID]", "serial": 1, "features": { - "deployment_history": {} + "deployment_history_best_effort": {} } } @@ -92,9 +92,18 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "lineage": "[UUID]", "serial": 1, "features": { - "deployment_history": {} + "deployment_history_best_effort": {} }, - "state": {} + "state": { + "resources.schemas.foo": { + "__id__": "main.dms_failed_recreate_schema_[UNIQUE_NAME]", + "state": { + "catalog_name": "main", + "comment": "v1", + "name": "dms_failed_recreate_schema_[UNIQUE_NAME]" + } + } + } } === A recreate deletes and then fails to create. The delete went through, so the resource really is gone: the failed operation clears its state (self-contained, not assuming the delete's own write reached the service), rather than describing the resource as it was before the deploy @@ -179,9 +188,9 @@ Files: 2 uploaded, 0 deleted "lineage": "[UUID]", "serial": 2, "features": { - "deployment_history": {} + "deployment_history_best_effort": {} }, "state": {} } -=== Planning from DMS state alone creates the schema, which is what has to happen: it no longer exists \ No newline at end of file +=== Planning after pulling remote file state creates the schema, which is what has to happen: it no longer exists \ No newline at end of file diff --git a/acceptance/bundle/dms/failed-recreate/script b/acceptance/bundle/dms/failed-recreate/script index 60bcc99df10..45d790615da 100644 --- a/acceptance/bundle/dms/failed-recreate/script +++ b/acceptance/bundle/dms/failed-recreate/script @@ -34,7 +34,7 @@ trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${d title "The raw state after failed recreate shows no resource" trace print_state.py --no-dms -title "Planning from DMS state alone creates the schema, which is what has to happen: it no longer exists" +title "Planning after pulling remote file state creates the schema, which is what has to happen: it no longer exists" rm -rf .databricks $CLI bundle plan &> LOG.plan_final # Ensure "$OUT_REQUESTS" is not left behind (RecordRequests creates it when running tests) diff --git a/acceptance/bundle/dms/failed-update-permissions/output.txt b/acceptance/bundle/dms/failed-update-permissions/output.txt index 47ef6cd6eac..99b4f03417c 100644 --- a/acceptance/bundle/dms/failed-update-permissions/output.txt +++ b/acceptance/bundle/dms/failed-update-permissions/output.txt @@ -15,7 +15,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "lineage": "[UUID]", "serial": 1, "features": { - "deployment_history": {} + "deployment_history_best_effort": {} } } @@ -45,9 +45,50 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged "lineage": "[UUID]", "serial": 1, "features": { - "deployment_history": {} + "deployment_history_best_effort": {} }, - "state": {} + "state": { + "resources.jobs.foo": { + "__id__": "[FOO_ID]", + "state": { + "deployment": { + "deployment_id": "[DEPLOYMENT_ID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-failed-update-permissions-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "foo", + "queue": { + "enabled": true + } + } + }, + "resources.jobs.foo.permissions": { + "__id__": "/jobs/[FOO_ID]", + "state": { + "object_id": "/jobs/[FOO_ID]", + "__embed__": [ + { + "level": "CAN_VIEW", + "user_name": "viewer@example.com" + }, + { + "level": "IS_OWNER", + "user_name": "[USERNAME]" + } + ] + }, + "depends_on": [ + { + "node": "resources.jobs.foo", + "label": "${resources.jobs.foo.id}" + } + ] + } + } } === A permissions update that fails records the failure with the permissions it still has. The resource ID here is a path rather than a number, and the state is the embedded permission list, so this covers a shape the resource tests do not @@ -100,9 +141,50 @@ Files: 1 uploaded, 0 deleted "lineage": "[UUID]", "serial": 2, "features": { - "deployment_history": {} + "deployment_history_best_effort": {} }, - "state": {} + "state": { + "resources.jobs.foo": { + "__id__": "[FOO_ID]", + "state": { + "deployment": { + "deployment_id": "[DEPLOYMENT_ID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-failed-update-permissions-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "foo", + "queue": { + "enabled": true + } + } + }, + "resources.jobs.foo.permissions": { + "__id__": "/jobs/[FOO_ID]", + "state": { + "object_id": "/jobs/[FOO_ID]", + "__embed__": [ + { + "level": "CAN_VIEW", + "user_name": "viewer@example.com" + }, + { + "level": "IS_OWNER", + "user_name": "[USERNAME]" + } + ] + }, + "depends_on": [ + { + "node": "resources.jobs.foo", + "label": "${resources.jobs.foo.id}" + } + ] + } + } } >>> print_requests.py --dms //api/2.0/bundle --oneline diff --git a/acceptance/bundle/dms/mismatch-before-read/output.txt b/acceptance/bundle/dms/mismatch-before-read/output.txt index d81bb0875eb..1f51ec1804a 100644 --- a/acceptance/bundle/dms/mismatch-before-read/output.txt +++ b/acceptance/bundle/dms/mismatch-before-read/output.txt @@ -8,7 +8,15 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> fault.py GET /api/2.0/bundle/deployments/* 500 0 1 >>> env DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false [CLI] bundle plan -Error: deployment history setting (false) does not match the existing state (true) +update jobs.foo -Update experimental.deployment_history to match the existing deployment, or run "databricks bundle destroy" to start over +Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged +>>> env DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-mismatch-before-read-[UNIQUE_NAME]/default/files... +Updated jobs.foo +Files: 1 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged + +>>> jq .features // {} .databricks/bundle/default/resources.json +{} diff --git a/acceptance/bundle/dms/mismatch-before-read/script b/acceptance/bundle/dms/mismatch-before-read/script index 8d82f0f9c95..a89228b2462 100644 --- a/acceptance/bundle/dms/mismatch-before-read/script +++ b/acceptance/bundle/dms/mismatch-before-read/script @@ -3,6 +3,9 @@ envsubst < databricks.yml.tmpl > databricks.yml trace $CLI bundle deploy update_file.py databricks.yml 'deployment_history: true' 'deployment_history: false' -# The configuration mismatch must be rejected before trying to read the deployment from DMS. +# Disabling file-backed history must not depend on DMS being available. +rm -f "$OUT_REQUESTS" trace fault.py "GET /api/2.0/bundle/deployments/*" 500 0 1 -musterr trace env DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false $CLI bundle plan +trace env DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false $CLI bundle plan +trace env DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false $CLI bundle deploy +trace jq '.features // {}' .databricks/bundle/default/resources.json diff --git a/acceptance/bundle/dms/multiple-resources/output.txt b/acceptance/bundle/dms/multiple-resources/output.txt index 10c8a496c2b..2979ca25346 100644 --- a/acceptance/bundle/dms/multiple-resources/output.txt +++ b/acceptance/bundle/dms/multiple-resources/output.txt @@ -25,7 +25,7 @@ Resources: 5 created, 0 changed, 0 deleted, 0 unchanged "lineage": "[UUID]", "serial": 1, "features": { - "deployment_history": {} + "deployment_history_best_effort": {} } } @@ -70,9 +70,100 @@ Plan: 0 to add, 0 to change, 0 to delete, 5 unchanged "lineage": "[UUID]", "serial": 1, "features": { - "deployment_history": {} + "deployment_history_best_effort": {} }, - "state": {} + "state": { + "resources.jobs.five": { + "__id__": "[FIVE_ID]", + "state": { + "deployment": { + "deployment_id": "[DEPLOYMENT_ID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "five", + "queue": { + "enabled": true + } + } + }, + "resources.jobs.four": { + "__id__": "[FOUR_ID]", + "state": { + "deployment": { + "deployment_id": "[DEPLOYMENT_ID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "four", + "queue": { + "enabled": true + } + } + }, + "resources.jobs.one": { + "__id__": "[ONE_ID]", + "state": { + "deployment": { + "deployment_id": "[DEPLOYMENT_ID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "one", + "queue": { + "enabled": true + } + } + }, + "resources.jobs.three": { + "__id__": "[THREE_ID]", + "state": { + "deployment": { + "deployment_id": "[DEPLOYMENT_ID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "three", + "queue": { + "enabled": true + } + } + }, + "resources.jobs.two": { + "__id__": "[TWO_ID]", + "state": { + "deployment": { + "deployment_id": "[DEPLOYMENT_ID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "two", + "queue": { + "enabled": true + } + } + } + } } === Redeploy with no changes: nothing is applied, so no version is created at all - the deployment's version tracks state changes, like serial @@ -89,9 +180,9 @@ Resources: 0 created, 0 changed, 0 deleted, 5 unchanged "plan_version": [PLAN_VERSION], "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", - "serial": 1, + "serial": 2, "features": { - "deployment_history": {} + "deployment_history_best_effort": {} } } @@ -134,9 +225,100 @@ Plan: 0 to add, 0 to change, 0 to delete, 5 unchanged "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", - "serial": 1, + "serial": 2, "features": { - "deployment_history": {} + "deployment_history_best_effort": {} }, - "state": {} + "state": { + "resources.jobs.five": { + "__id__": "[FIVE_ID]", + "state": { + "deployment": { + "deployment_id": "[DEPLOYMENT_ID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "five", + "queue": { + "enabled": true + } + } + }, + "resources.jobs.four": { + "__id__": "[FOUR_ID]", + "state": { + "deployment": { + "deployment_id": "[DEPLOYMENT_ID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "four", + "queue": { + "enabled": true + } + } + }, + "resources.jobs.one": { + "__id__": "[ONE_ID]", + "state": { + "deployment": { + "deployment_id": "[DEPLOYMENT_ID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "one", + "queue": { + "enabled": true + } + } + }, + "resources.jobs.three": { + "__id__": "[THREE_ID]", + "state": { + "deployment": { + "deployment_id": "[DEPLOYMENT_ID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "three", + "queue": { + "enabled": true + } + } + }, + "resources.jobs.two": { + "__id__": "[TWO_ID]", + "state": { + "deployment": { + "deployment_id": "[DEPLOYMENT_ID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "two", + "queue": { + "enabled": true + } + } + } + } } diff --git a/acceptance/bundle/dms/no-drift/output.txt b/acceptance/bundle/dms/no-drift/output.txt index c2fe06a747e..f19d44a65f0 100644 --- a/acceptance/bundle/dms/no-drift/output.txt +++ b/acceptance/bundle/dms/no-drift/output.txt @@ -35,7 +35,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "lineage": "[UUID]", "serial": 1, "features": { - "deployment_history": {} + "deployment_history_best_effort": {} } } @@ -50,9 +50,52 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged "lineage": "[UUID]", "serial": 1, "features": { - "deployment_history": {} + "deployment_history_best_effort": {} }, - "state": {} + "state": { + "resources.jobs.foo": { + "__id__": "[FOO_ID]", + "state": { + "deployment": { + "deployment_id": "[DEPLOYMENT_ID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "foo", + "queue": { + "enabled": true + } + } + }, + "resources.pipelines.bar": { + "__id__": "[BAR_ID]", + "state": { + "catalog": "main", + "channel": "CURRENT", + "deployment": { + "deployment_id": "[DEPLOYMENT_ID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edition": "ADVANCED", + "libraries": [ + { + "file": { + "path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/files/transform.py" + } + } + ], + "name": "bar-[UNIQUE_NAME]", + "schema": "default", + "serverless": true + } + } + } } === Plan without touching anything: the deployment stamp must not show as drift @@ -120,7 +163,7 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged } { "method": "PUT", - "path": "/api/2.0/pipelines/[UUID]", + "path": "/api/2.0/pipelines/[BAR_ID]", "body": { "catalog": "main", "channel": "CURRENT", @@ -181,7 +224,7 @@ Warning: The "development" property of pipelines is deprecated. Use "mode: devel "lineage": "[UUID]", "serial": 2, "features": { - "deployment_history": {} + "deployment_history_best_effort": {} } } @@ -200,9 +243,53 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged "lineage": "[UUID]", "serial": 2, "features": { - "deployment_history": {} + "deployment_history_best_effort": {} }, - "state": {} + "state": { + "resources.jobs.foo": { + "__id__": "[FOO_ID]", + "state": { + "deployment": { + "deployment_id": "[DEPLOYMENT_ID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "foo", + "queue": { + "enabled": true + } + } + }, + "resources.pipelines.bar": { + "__id__": "[BAR_ID]", + "state": { + "catalog": "main", + "channel": "CURRENT", + "deployment": { + "deployment_id": "[DEPLOYMENT_ID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "2" + }, + "development": true, + "edition": "ADVANCED", + "libraries": [ + { + "file": { + "path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/files/transform.py" + } + } + ], + "name": "bar-[UNIQUE_NAME]", + "schema": "default", + "serverless": true + } + } + } } === Plan again: no residual drift, and the version stamps are ignored diff --git a/acceptance/bundle/dms/no-drift/script b/acceptance/bundle/dms/no-drift/script index 66898d04cf9..503931ed646 100644 --- a/acceptance/bundle/dms/no-drift/script +++ b/acceptance/bundle/dms/no-drift/script @@ -2,6 +2,7 @@ envsubst < databricks.yml.tmpl > databricks.yml title "Deploy job foo and pipeline bar; both are recorded at version 1" trace $CLI bundle deploy +replace_ids.py deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-no-drift-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') add_repl "$deployment_id" DEPLOYMENT_ID diff --git a/acceptance/bundle/dms/no-resources/output.txt b/acceptance/bundle/dms/no-resources/output.txt index 884c22c6583..6c44aa176ab 100644 --- a/acceptance/bundle/dms/no-resources/output.txt +++ b/acceptance/bundle/dms/no-resources/output.txt @@ -35,8 +35,9 @@ deployment_history_enabled true "plan_version": [PLAN_VERSION], "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", + "serial": 1, "features": { - "deployment_history": {} + "deployment_history_best_effort": {} } } @@ -53,8 +54,9 @@ Plan: 0 to add, 0 to change, 0 to delete, 0 unchanged "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", + "serial": 1, "features": { - "deployment_history": {} + "deployment_history_best_effort": {} }, "state": {} } @@ -70,10 +72,6 @@ Resources: 0 created, 0 changed, 0 deleted, 0 unchanged "method": "GET", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]" } -{ - "method": "GET", - "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/resources" -} { "method": "GET", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]" @@ -82,18 +80,10 @@ Resources: 0 created, 0 changed, 0 deleted, 0 unchanged "method": "GET", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/resources" } -{ - "method": "GET", - "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/resources" -} { "method": "GET", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]" } -{ - "method": "GET", - "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/resources" -} === The plan records the deployment_history feature >>> [CLI] bundle plan -o json @@ -101,8 +91,9 @@ Resources: 0 created, 0 changed, 0 deleted, 0 unchanged "plan_version": [PLAN_VERSION], "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", + "serial": 2, "features": { - "deployment_history": {} + "deployment_history_best_effort": {} } } @@ -119,8 +110,9 @@ Plan: 0 to add, 0 to change, 0 to delete, 0 unchanged "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", + "serial": 2, "features": { - "deployment_history": {} + "deployment_history_best_effort": {} }, "state": {} } diff --git a/acceptance/bundle/dms/operation-upload-fails-once/output.txt b/acceptance/bundle/dms/operation-upload-fails-once/output.txt index dc976dadead..d64e6d62020 100644 --- a/acceptance/bundle/dms/operation-upload-fails-once/output.txt +++ b/acceptance/bundle/dms/operation-upload-fails-once/output.txt @@ -14,7 +14,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "lineage": "[UUID]", "serial": 1, "features": { - "deployment_history": {} + "deployment_history_best_effort": {} } } @@ -39,9 +39,18 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "lineage": "[UUID]", "serial": 1, "features": { - "deployment_history": {} + "deployment_history_best_effort": {} }, - "state": {} + "state": { + "resources.schemas.foo": { + "__id__": "main.dms_upload_fails_once_schema_[UNIQUE_NAME]", + "state": { + "catalog_name": "main", + "comment": "v1", + "name": "dms_upload_fails_once_schema_[UNIQUE_NAME]" + } + } + } } === A recreate records twice, and the first one fails: the second must still carry the sequence id staging left, because a rejected update earned no new one @@ -49,20 +58,15 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged >>> update_file.py databricks.yml catalog_name: main catalog_name: dms_other_[UNIQUE_NAME] ->>> musterr [CLI] bundle deploy --auto-approve +>>> [CLI] bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-upload-fails-once-[UNIQUE_NAME]/default/files... This action will result in the deletion or recreation of the following UC schemas. Any underlying data may be lost: recreate resources.schemas.foo Recreated schemas.foo -Error: recording operation for resources.schemas.foo: Fault injected by test. (500 INJECTED) - -Endpoint: PATCH [DATABRICKS_URL]/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/2/operations/schemas.foo?update_mask=state%2Cerror_message%2Cstatus -HTTP Status: 500 Internal Server Error -API error_code: INJECTED -API message: Fault injected by test. - +Warn: Failed to record deployment history: recording operation for resources.schemas.foo: Fault injected by test. Files: 2 uploaded, 0 deleted +Resources: 1 created, 0 changed, 1 deleted, 0 unchanged === The service lists the schema with the failure >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[DEPLOYMENT_ID] -o json @@ -82,9 +86,18 @@ Files: 2 uploaded, 0 deleted "lineage": "[UUID]", "serial": 2, "features": { - "deployment_history": {} + "deployment_history_best_effort": {} }, - "state": {} + "state": { + "resources.schemas.foo": { + "__id__": "dms_other_[UNIQUE_NAME].dms_upload_fails_once_schema_[UNIQUE_NAME]", + "state": { + "catalog_name": "dms_other_[UNIQUE_NAME]", + "comment": "v1", + "name": "dms_upload_fails_once_schema_[UNIQUE_NAME]" + } + } + } } >>> print_requests.py --dms //api/2.0/bundle @@ -139,3 +152,25 @@ Files: 2 uploaded, 0 deleted "completion_reason": "VERSION_COMPLETE_FAILURE" } } + +=== A completion failure does not block destroy or remote file cleanup +>>> fault.py POST /api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/3/complete 400 0 1 + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.schemas.foo + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.foo + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-upload-fails-once-[UNIQUE_NAME]/default + +Warn: Failed to complete deployment history version: Fault injected by test. +Destroy: 1 deleted + +>>> print_requests.py --dms //versions/3 --oneline +{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/3/operations/schemas.foo", "q": {"update_mask": "state,error_message,status"}, "body": {"error_message": "", "sequence_id": 0, "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/3/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} + +>>> musterr env MSYS_NO_PATHCONV=1 [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-upload-fails-once-[UNIQUE_NAME]/default/state/resources.json +Error: Path (/Workspace/Users/[USERNAME]/.bundle/dms-upload-fails-once-[UNIQUE_NAME]/default/state/resources.json) doesn't exist. diff --git a/acceptance/bundle/dms/operation-upload-fails-once/script b/acceptance/bundle/dms/operation-upload-fails-once/script index 2b539134571..69dab957164 100644 --- a/acceptance/bundle/dms/operation-upload-fails-once/script +++ b/acceptance/bundle/dms/operation-upload-fails-once/script @@ -22,7 +22,7 @@ other_catalog="dms_other_${UNIQUE_NAME}" $CLI catalogs create "$other_catalog" &> LOG.catalog trap '$CLI catalogs delete "$other_catalog" --force >> LOG.catalog 2>&1' EXIT trace update_file.py databricks.yml "catalog_name: main" "catalog_name: ${other_catalog}" -trace musterr $CLI bundle deploy --auto-approve +trace $CLI bundle deploy --auto-approve title "The service lists the schema with the failure" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-upload-fails-once-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') @@ -32,3 +32,10 @@ title "The raw recorded state after the failure" trace print_state.py --no-dms trace print_requests.py --dms //api/2.0/bundle + +title "A completion failure does not block destroy or remote file cleanup" +trace fault.py "POST /api/2.0/bundle/deployments/${deployment_id}/versions/3/complete" 400 0 1 +trace $CLI bundle destroy --auto-approve +trace print_requests.py --dms //versions/3 --oneline +trace musterr env MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-upload-fails-once-${UNIQUE_NAME}/default/state/resources.json" +rm -f "$OUT_REQUESTS" diff --git a/acceptance/bundle/dms/plan-without-dms/output.txt b/acceptance/bundle/dms/plan-without-dms/output.txt index ce1135f624d..db3e622639d 100644 --- a/acceptance/bundle/dms/plan-without-dms/output.txt +++ b/acceptance/bundle/dms/plan-without-dms/output.txt @@ -6,7 +6,7 @@ } === Enable DMS recording and try to deploy the non-DMS plan ->>> musterr [CLI] bundle deploy --plan tmp.plan.json +>>> musterr env DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true [CLI] bundle deploy --plan tmp.plan.json Error: this plan was created for a different set of state features than the target now has; run 'bundle plan' again diff --git a/acceptance/bundle/dms/plan-without-dms/script b/acceptance/bundle/dms/plan-without-dms/script index de35616b74e..8008168e6d9 100644 --- a/acceptance/bundle/dms/plan-without-dms/script +++ b/acceptance/bundle/dms/plan-without-dms/script @@ -5,11 +5,7 @@ $CLI bundle plan -o json > tmp.plan.json trace jq '{features}' < tmp.plan.json title "Enable DMS recording and try to deploy the non-DMS plan" -update_file.py databricks.yml 'bundle:' 'experimental: - deployment_history: true - -bundle:' -trace musterr $CLI bundle deploy --plan tmp.plan.json +trace musterr env DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle deploy --plan tmp.plan.json trace print_requests.py //api/2.0/bundle --get rm -f tmp.plan.json "$OUT_REQUESTS" diff --git a/acceptance/bundle/dms/provenance/output.txt b/acceptance/bundle/dms/provenance/output.txt index 4decd10218b..71383501895 100644 --- a/acceptance/bundle/dms/provenance/output.txt +++ b/acceptance/bundle/dms/provenance/output.txt @@ -59,7 +59,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "lineage": "[UUID]", "serial": 1, "features": { - "deployment_history": {} + "deployment_history_best_effort": {} } } @@ -100,7 +100,29 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "lineage": "[UUID]", "serial": 1, "features": { - "deployment_history": {} + "deployment_history_best_effort": {} }, - "state": {} + "state": { + "resources.jobs.foo": { + "__id__": "[FOO_ID]", + "state": { + "deployment": { + "deployment_id": "[DEPLOYMENT_ID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-provenance-[UNIQUE_NAME]/dev/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 4, + "name": "[dev [USERNAME]] foo", + "queue": { + "enabled": true + }, + "tags": { + "dev": "[USERNAME]" + } + } + } + } } diff --git a/acceptance/bundle/dms/record-failure/output.txt b/acceptance/bundle/dms/record-failure/output.txt index 9f61f4a9274..50042925eda 100644 --- a/acceptance/bundle/dms/record-failure/output.txt +++ b/acceptance/bundle/dms/record-failure/output.txt @@ -18,7 +18,7 @@ Files: 2 uploaded, 0 deleted "lineage": "[UUID]", "serial": 1, "features": { - "deployment_history": {} + "deployment_history_best_effort": {} } } @@ -92,12 +92,12 @@ Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged "lineage": "[UUID]", "serial": 1, "features": { - "deployment_history": {} + "deployment_history_best_effort": {} }, "state": {} } -=== Redeploying from DMS state alone still creates the failed resource: it was never applied, so it must not be skipped as unchanged +=== Redeploying after pulling remote file state still creates the failed resource: it was never applied, so it must not be skipped as unchanged >>> [CLI] bundle plan create jobs.doomed diff --git a/acceptance/bundle/dms/record-failure/script b/acceptance/bundle/dms/record-failure/script index 806e7ab8b15..7e973b4b847 100644 --- a/acceptance/bundle/dms/record-failure/script +++ b/acceptance/bundle/dms/record-failure/script @@ -18,7 +18,7 @@ trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${d trace print_state.py --no-dms -title "Redeploying from DMS state alone still creates the failed resource: it was never applied, so it must not be skipped as unchanged" +title "Redeploying after pulling remote file state still creates the failed resource: it was never applied, so it must not be skipped as unchanged" rm -rf .databricks trace $CLI bundle plan # plan probes the state files concurrently, so the recorded order varies. Only the diff --git a/acceptance/bundle/dms/stale-local-state/databricks.yml.tmpl b/acceptance/bundle/dms/stale-local-state/databricks.yml.tmpl index a9b1eada1f9..564b0d03c87 100644 --- a/acceptance/bundle/dms/stale-local-state/databricks.yml.tmpl +++ b/acceptance/bundle/dms/stale-local-state/databricks.yml.tmpl @@ -1,9 +1,6 @@ bundle: name: dms-stale-local-state-$UNIQUE_NAME -experimental: - deployment_history: true - resources: jobs: foo: diff --git a/acceptance/bundle/dms/stale-local-state/output.txt b/acceptance/bundle/dms/stale-local-state/output.txt index 5c2b2225a84..be6f19f4d88 100644 --- a/acceptance/bundle/dms/stale-local-state/output.txt +++ b/acceptance/bundle/dms/stale-local-state/output.txt @@ -4,35 +4,41 @@ {"serial":1,"features":null} >>> jq -c {serial, features} .databricks/resources.v2.json -{"serial":2,"features":{"deployment_history":{}}} +{"serial":2,"features":{"deployment_history_best_effort":{}}} -=== Local recorded serial 3 ahead of DMS version 2 is rejected +=== File serial 3 ahead of DMS version 2 is allowed >>> update_file.py .databricks/bundle/default/resources.json "serial": 2 "serial": 3 ->>> musterr [CLI] bundle deploy -Error: inconsistent deployment state: resource state serial (3) is ahead of deployment history version (2) +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged - -=== 1. Config false and env false: newer remote state replaces the local copy and reads DMS +=== 1. Config false and env false: newer remote file state replaces the local copy >>> cp .databricks/resources.v1.json .databricks/bundle/default/resources.json >>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false [CLI] bundle summary --force-pull -o json -Warn: Deployment history setting (false) does not match the existing state (true). Using the existing state. {"job_id":"[FOO_ID]"} >>> cmp .databricks/resources.v2.json .databricks/bundle/default/resources.json >>> print_requests.py --dms //resources --get --oneline -{"method": "GET", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/resources"} -=== 2. Config removed and env false: restore the same snapshot, then read remote DMS state +=== 2. Config removed and env false: restore the same snapshot, then read the remote file state >>> cp .databricks/resources.v1.json .databricks/bundle/default/resources.json >>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false [CLI] bundle summary --force-pull -o json -Warn: Deployment history setting (false) does not match the existing state (true). Using the existing state. {"job_id":"[FOO_ID]"} >>> cmp .databricks/resources.v2.json .databricks/bundle/default/resources.json >>> print_requests.py --dms //resources --get --oneline -{"method": "GET", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/resources"} + +=== Destroy follows newer local state even when configuration enables history +>>> env DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.foo + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-stale-local-state-[UNIQUE_NAME]/default + +Destroy: 1 deleted + +>>> print_requests.py --dms //api/2.0/bundle --oneline diff --git a/acceptance/bundle/dms/stale-local-state/script b/acceptance/bundle/dms/stale-local-state/script index 38d0a3725b4..7f0cfbf9663 100644 --- a/acceptance/bundle/dms/stale-local-state/script +++ b/acceptance/bundle/dms/stale-local-state/script @@ -2,37 +2,45 @@ envsubst < databricks.yml.tmpl > databricks.yml title "Setup (LOG.setup): reconstruct full version 1 without a history marker, then deploy version 2" { - $CLI bundle deploy + DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle deploy DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true print_state.py > .databricks/resources.v1.json DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true replace_ids.py update_file.py databricks.yml 'name: foo' 'name: updated' - $CLI bundle deploy + DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle deploy cp .databricks/bundle/default/resources.json .databricks/resources.v2.json deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-stale-local-state-${UNIQUE_NAME}/default/state/resources.deployment.json" -o json | jq -r '.object_id') add_repl "$deployment_id" DEPLOYMENT_ID } &> LOG.setup +# The environment flag records best-effort history; YAML selects the same mode. +update_file.py databricks.yml 'bundle:' 'experimental: + deployment_history: true + +bundle:' trace jq -c '{serial, features}' .databricks/resources.v1.json trace jq -c '{serial, features}' .databricks/resources.v2.json -title "Local recorded serial 3 ahead of DMS version 2 is rejected" +title "File serial 3 ahead of DMS version 2 is allowed" trace update_file.py .databricks/bundle/default/resources.json '"serial": 2' '"serial": 3' -trace musterr $CLI bundle deploy +trace $CLI bundle plan rm -f "$OUT_REQUESTS" -title "1. Config false and env false: newer remote state replaces the local copy and reads DMS" +title "1. Config false and env false: newer remote file state replaces the local copy" update_file.py databricks.yml 'deployment_history: true' 'deployment_history: false' trace cp .databricks/resources.v1.json .databricks/bundle/default/resources.json trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false $CLI bundle summary --force-pull -o json | jq -c '{job_id: .resources.jobs.foo.id}' trace cmp .databricks/resources.v2.json .databricks/bundle/default/resources.json trace print_requests.py --dms //resources --get --oneline -title "2. Config removed and env false: restore the same snapshot, then read remote DMS state" +title "2. Config removed and env false: restore the same snapshot, then read the remote file state" update_file.py databricks.yml $'experimental:\n deployment_history: false\n\n' '' trace cp .databricks/resources.v1.json .databricks/bundle/default/resources.json trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false $CLI bundle summary --force-pull -o json | jq -c '{job_id: .resources.jobs.foo.id}' trace cmp .databricks/resources.v2.json .databricks/bundle/default/resources.json trace print_requests.py --dms //resources --get --oneline -DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle destroy --auto-approve &> LOG.cleanup +title "Destroy follows newer local state even when configuration enables history" +jq 'del(.features) | .serial = 3' .databricks/resources.v2.json > .databricks/bundle/default/resources.json +trace env DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle destroy --auto-approve +trace print_requests.py --dms //api/2.0/bundle --oneline rm -f "$OUT_REQUESTS" diff --git a/acceptance/bundle/dms/stale-plan/output.txt b/acceptance/bundle/dms/stale-plan/output.txt index e9fbc70f832..eb46510cdf1 100644 --- a/acceptance/bundle/dms/stale-plan/output.txt +++ b/acceptance/bundle/dms/stale-plan/output.txt @@ -5,7 +5,7 @@ "plan_version": [PLAN_VERSION], "cli_version": "[CLI_VERSION]", "features": { - "deployment_history": {} + "deployment_history_best_effort": {} } } @@ -35,9 +35,28 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "lineage": "[UUID]", "serial": 1, "features": { - "deployment_history": {} + "deployment_history_best_effort": {} }, - "state": {} + "state": { + "resources.jobs.foo": { + "__id__": "[FOO_ID]", + "state": { + "deployment": { + "deployment_id": "[DEPLOYMENT_ID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-stale-plan-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "foo", + "queue": { + "enabled": true + } + } + } + } } === A saved plan carries the version it was built against diff --git a/acceptance/bundle/dms/state-features/output.txt b/acceptance/bundle/dms/state-features/output.txt index 342fbfe6a44..92220c5f72a 100644 --- a/acceptance/bundle/dms/state-features/output.txt +++ b/acceptance/bundle/dms/state-features/output.txt @@ -1,5 +1,5 @@ -=== Deploy records version 3 and the deployment_history feature, with resources held in DMS +=== Deploy keeps resource state in the file and enforces best-effort history >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-state-features-[UNIQUE_NAME]/default/files... Created jobs.foo @@ -10,31 +10,58 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged { "state_version": 3, "features": { - "deployment_history": {} + "deployment_history_best_effort": {} }, - "state": {} + "state": { + "resources.jobs.foo": { + "__id__": "[FOO_ID]", + "state": { + "deployment": { + "deployment_id": "[DEPLOYMENT_ID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-state-features-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "state-features", + "queue": { + "enabled": true + } + } + } + } } -=== Reopening version 3 reads resources from DMS and produces a no-op plan +=== Reopening reads resources from the file and produces a no-op plan >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged >>> print_requests.py --dms //resources --get --oneline -{"method": "GET", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/resources"} -=== Disabling deployment history is refused by plan and deploy, while destroy keeps using DMS from state +=== Disabling deployment history allows plan and deploy, and removes the file marker >>> env DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false [CLI] bundle plan -Error: deployment history setting (false) does not match the existing state (true) - -Update experimental.deployment_history to match the existing deployment, or run "databricks bundle destroy" to start over +update jobs.foo +Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged >>> env DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false [CLI] bundle deploy -Error: deployment history setting (false) does not match the existing state (true) +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-state-features-[UNIQUE_NAME]/default/files... +Updated jobs.foo +Files: 1 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged + +>>> jq .features // {} .databricks/bundle/default/resources.json +{} -Update experimental.deployment_history to match the existing deployment, or run "databricks bundle destroy" to start over +>>> print_requests.py --dms //api/2.0/bundle --oneline +=== Removing the YAML setting also leaves history disabled +>>> env DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged +=== Destroy also uses the disabled setting >>> env DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.jobs.foo @@ -44,7 +71,3 @@ All files and directories at the following location will be deleted: /Workspace/ Destroy: 1 deleted >>> print_requests.py --dms //api/2.0/bundle --oneline -{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions", "q": {"version_id": "2"}, "body": {"cli_version": "[CLI_VERSION]", "operations": [{"action_type": "OPERATION_ACTION_TYPE_DELETE", "resource_key": "jobs.foo"}], "previous_version_id": "1", "version_type": "VERSION_TYPE_DESTROY"}} -{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/2/operations/jobs.foo", "q": {"update_mask": "state,error_message,status"}, "body": {"error_message": "", "sequence_id": 0, "status": "OPERATION_STATUS_SUCCEEDED"}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/2/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} -{"method": "DELETE", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]"} diff --git a/acceptance/bundle/dms/state-features/script b/acceptance/bundle/dms/state-features/script index 349845853cc..0f632d18fc0 100644 --- a/acceptance/bundle/dms/state-features/script +++ b/acceptance/bundle/dms/state-features/script @@ -1,21 +1,30 @@ envsubst < databricks.yml.tmpl > databricks.yml -title "Deploy records version 3 and the deployment_history feature, with resources held in DMS" +title "Deploy keeps resource state in the file and enforces best-effort history" trace $CLI bundle deploy +replace_ids.py trace jq '{state_version, features, state}' .databricks/bundle/default/resources.json deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-state-features-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') add_repl "$deployment_id" DEPLOYMENT_ID rm -f "$OUT_REQUESTS" -title "Reopening version 3 reads resources from DMS and produces a no-op plan" +title "Reopening reads resources from the file and produces a no-op plan" trace $CLI bundle plan trace print_requests.py --dms //resources --get --oneline -title "Disabling deployment history is refused by plan and deploy, while destroy keeps using DMS from state" +title "Disabling deployment history allows plan and deploy, and removes the file marker" update_file.py databricks.yml 'deployment_history: true' 'deployment_history: false' for command in plan deploy; do - musterr trace env DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false $CLI bundle "$command" + trace env DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false $CLI bundle "$command" done +trace jq '.features // {}' .databricks/bundle/default/resources.json +trace print_requests.py --dms //api/2.0/bundle --oneline + +title "Removing the YAML setting also leaves history disabled" +update_file.py databricks.yml $'experimental:\n deployment_history: false\n' '' +trace env DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false $CLI bundle plan + +title "Destroy also uses the disabled setting" trace env DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false $CLI bundle destroy --auto-approve trace print_requests.py --dms //api/2.0/bundle --oneline diff --git a/acceptance/bundle/dms/successful-recreate/output.txt b/acceptance/bundle/dms/successful-recreate/output.txt index f2ecc3f8cbc..016b2cc5eba 100644 --- a/acceptance/bundle/dms/successful-recreate/output.txt +++ b/acceptance/bundle/dms/successful-recreate/output.txt @@ -67,7 +67,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "lineage": "[UUID]", "serial": 1, "features": { - "deployment_history": {} + "deployment_history_best_effort": {} } } @@ -92,9 +92,18 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "lineage": "[UUID]", "serial": 1, "features": { - "deployment_history": {} + "deployment_history_best_effort": {} }, - "state": {} + "state": { + "resources.schemas.foo": { + "__id__": "main.dms_successful_recreate_schema_[UNIQUE_NAME]", + "state": { + "catalog_name": "main", + "comment": "v1", + "name": "dms_successful_recreate_schema_[UNIQUE_NAME]" + } + } + } } === Recreate writes state twice - the entry is dropped, then the new resource is saved @@ -170,7 +179,7 @@ Resources: 1 created, 0 changed, 1 deleted, 0 unchanged "lineage": "[UUID]", "serial": 2, "features": { - "deployment_history": {} + "deployment_history_best_effort": {} } } @@ -195,9 +204,18 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "lineage": "[UUID]", "serial": 2, "features": { - "deployment_history": {} + "deployment_history_best_effort": {} }, - "state": {} + "state": { + "resources.schemas.foo": { + "__id__": "dms_other_[UNIQUE_NAME].dms_successful_recreate_schema_[UNIQUE_NAME]", + "state": { + "catalog_name": "dms_other_[UNIQUE_NAME]", + "comment": "v1", + "name": "dms_successful_recreate_schema_[UNIQUE_NAME]" + } + } + } } === Destroy: the delete is recorded with the id and no state diff --git a/acceptance/bundle/dms/version-never-created/output.txt b/acceptance/bundle/dms/version-never-created/output.txt index 989744df818..53bc531b5ab 100644 --- a/acceptance/bundle/dms/version-never-created/output.txt +++ b/acceptance/bundle/dms/version-never-created/output.txt @@ -1,29 +1,25 @@ -=== The first version fails, so no deployment record exists - only the node naming its ID ->>> musterr [CLI] bundle deploy +=== Version creation fails, but file-backed deployment still succeeds +>>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/files... -Error: failed to create deployment version: Internal error (500 INTERNAL_ERROR) - -Endpoint: POST [DATABRICKS_URL]/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions?version_id=1 -HTTP Status: 500 Internal Server Error -API error_code: INTERNAL_ERROR -API message: Internal error - +Warn: Could not record deployment history; continuing with file state: failed to create deployment version: Internal error +Created jobs.foo Files: 1 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle plan -o json { "plan_version": [PLAN_VERSION], "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 1, "features": { - "deployment_history": {} + "deployment_history_best_effort": {} } } >>> [CLI] bundle plan -create jobs.foo - -Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged >>> MSYS_NO_PATHCONV=1 [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/state/resources.deployment.json { @@ -34,27 +30,22 @@ Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged {"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"display_name": "dms-version-never-created", "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/state", "target_name": "default", "workspace_info": {"file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default"}}} {"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "operations": [{"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_key": "jobs.foo"}], "version_type": "VERSION_TYPE_DEPLOY"}} {"method": "GET", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]"} -{"method": "GET", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/resources"} {"method": "GET", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]"} -{"method": "GET", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/resources"} >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[DEPLOYMENT_ID] -o json [] -=== The next deploy reuses the ID that node names and retries version 1, rather than creating a second deployment ->>> musterr [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/files... -Error: failed to create deployment version: Internal error (500 INTERNAL_ERROR) +=== The next resource update also succeeds while history version creation fails +>>> update_file.py databricks.yml name: foo name: renamed -Endpoint: POST [DATABRICKS_URL]/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions?version_id=1 -HTTP Status: 500 Internal Server Error -API error_code: INTERNAL_ERROR -API message: Internal error - -Files: 0 uploaded, 0 deleted +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/files... +Warn: Could not record deployment history; continuing with file state: failed to create deployment version: Internal error +Updated jobs.foo +Files: 1 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged >>> print_requests.py --dms //api/2.0/bundle --get --oneline {"method": "GET", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/resources"} {"method": "GET", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]"} -{"method": "GET", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/resources"} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "operations": [{"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_key": "jobs.foo"}], "version_type": "VERSION_TYPE_DEPLOY"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions", "q": {"version_id": "2"}, "body": {"cli_version": "[CLI_VERSION]", "operations": [{"action_type": "OPERATION_ACTION_TYPE_UPDATE", "resource_key": "jobs.foo"}], "version_type": "VERSION_TYPE_DEPLOY"}} diff --git a/acceptance/bundle/dms/version-never-created/script b/acceptance/bundle/dms/version-never-created/script index 9498fa05eff..5601cb2129b 100644 --- a/acceptance/bundle/dms/version-never-created/script +++ b/acceptance/bundle/dms/version-never-created/script @@ -1,5 +1,5 @@ -title "The first version fails, so no deployment record exists - only the node naming its ID" -trace musterr $CLI bundle deploy +title "Version creation fails, but file-backed deployment still succeeds" +trace $CLI bundle deploy dms_plan trace MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-version-never-created/default/state/resources.deployment.json" | jq '{object_type}' trace print_requests.py --dms //api/2.0/bundle --get --oneline @@ -8,6 +8,7 @@ deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/$ add_repl "$deployment_id" DEPLOYMENT_ID trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' -title "The next deploy reuses the ID that node names and retries version 1, rather than creating a second deployment" -trace musterr $CLI bundle deploy +title "The next resource update also succeeds while history version creation fails" +trace update_file.py databricks.yml 'name: foo' 'name: renamed' +trace $CLI bundle deploy trace print_requests.py --dms //api/2.0/bundle --get --oneline diff --git a/bundle/config/experimental.go b/bundle/config/experimental.go index 6fd5511b79b..2df01b7e226 100644 --- a/bundle/config/experimental.go +++ b/bundle/config/experimental.go @@ -54,8 +54,9 @@ type Experimental struct { // service (DMS), which records deployment history and tracks what changed // across deployments. // - // Only for bundles with no deployed resources yet: DMS becomes the source of - // truth for their state. + // Resource state remains in resources.json. History uploads are best effort: + // DMS errors do not fail file-backed deployments. + // Existing DMS-authoritative deployments keep their current state source. // An explicit value takes priority over the environment setting. DeploymentHistory *bool `json:"deployment_history,omitempty"` } diff --git a/bundle/direct/apply.go b/bundle/direct/apply.go index d4f4a2ffb41..9cdd5a4f505 100644 --- a/bundle/direct/apply.go +++ b/bundle/direct/apply.go @@ -265,7 +265,7 @@ func (d *DeploymentUnit) Delete(ctx context.Context, db *dstate.DeploymentState, log.Warnf(ctx, "Ignoring permission error when deleting %s id=%s: %s", d.ResourceKey, oldID, err) } else if d.deleteConfirmedGone(ctx, oldID) { log.Warnf(ctx, "Treating %s id=%s as already deleted despite delete error: %s", d.ResourceKey, oldID, err) - } else if db.IsDeploymentMetadataService() { + } else if db.RecordsHistory() { // When using the deployment metadata service, record the error. err = fmt.Errorf("deleting id=%s: %w", oldID, err) db.RecordFailure(d.ResourceKey, oldID, err) diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index 411eeb2af3d..a8e2a441dda 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -448,7 +448,7 @@ func (db *DeploymentState) StateFeatures() map[string]struct{} { } // GetSerial reports the counter used to validate saved plans. -// DMS modes use the service version; plain file modes use the file serial. +// WritingDMS uses the service version; plain file modes use the file serial. // File modes with history use the larger counter because either can advance independently. func (db *DeploymentState) GetSerial() int { db.mu.Lock() @@ -456,7 +456,7 @@ func (db *DeploymentState) GetSerial() int { switch db.mode { case WritingDMS: return db.VersionID - case WritingFileWithHistory, WritingFileWithHistoryEnforced: + case ReadingFile, WritingFileWithHistory, WritingFileWithHistoryEnforced: return max(db.VersionID, db.Data.Serial) default: return db.Data.Serial @@ -477,11 +477,14 @@ func (db *DeploymentState) IsDeploymentMetadataService() bool { return db.isDeploymentMetadataService() } -// RecordsHistory reports whether this mode records resource operations with DMS. +// RecordsHistory reports whether the loaded file state or selected write mode records DMS history. func (db *DeploymentState) RecordsHistory() bool { db.mu.Lock() defer db.mu.Unlock() switch db.mode { + case ReadingFile: + _, history := db.Data.Features[FeatureDeploymentHistoryBestEffort] + return history case WritingDMS, WritingFileWithHistory, WritingFileWithHistoryEnforced: return true default: @@ -809,9 +812,9 @@ func (db *DeploymentState) mergeWalIntoState(ctx context.Context) (bool, error) } hasEntries := lineNumber > 1 - // Enforced history can change the header's features without writing resource entries. - // Persist that marker too, before removing the WAL. - if lineNumber == 1 && !maps.Equal(db.Data.Features, newFeatures) { + // Persist enforced-history headers even without resource writes. Configured features + // may already match in memory while the file still carries different markers. + if lineNumber == 1 && (db.mode == WritingFileWithHistoryEnforced || !maps.Equal(db.Data.Features, newFeatures)) { hasEntries = true } @@ -898,6 +901,16 @@ func (db *DeploymentState) Finalize(ctx context.Context) (resourcestate.Exported // UpgradeToWrite opens a WAL for file state without re-reading resources. // DMS and file-history modes are already writable from Open, so upgrading them is a no-op. func (db *DeploymentState) UpgradeToWrite() error { + return db.upgradeToWrite(WritingFile) +} + +// UpgradeToWriteWithHistory enables enforced file-backed history after approval. +// Reads still use the same file state; only the write policy changes. +func (db *DeploymentState) UpgradeToWriteWithHistory() error { + return db.upgradeToWrite(WritingFileWithHistoryEnforced) +} + +func (db *DeploymentState) upgradeToWrite(mode Mode) error { db.mu.Lock() defer db.mu.Unlock() @@ -910,7 +923,7 @@ func (db *DeploymentState) UpgradeToWrite() error { case WritingFile: return errors.New("internal error: DeploymentState is already open for write") case ReadingFile: - db.mode = WritingFile + db.mode = mode if err := db.setupWal(); err != nil { db.mode = ReadingFile return err @@ -943,11 +956,16 @@ func (db *DeploymentState) setupWal() error { CLIVersion: build.GetInfo().Version, Features: maps.Clone(db.Data.Features), } - if db.mode == WritingFileWithHistoryEnforced { + switch db.mode { + case WritingFileWithHistoryEnforced: if walHead.Features == nil { walHead.Features = make(map[string]struct{}) } walHead.Features[FeatureDeploymentHistoryBestEffort] = struct{}{} + case WritingFile: + walHead.Features = map[string]struct{}{} + default: + // Other modes preserve the existing markers. } if err := appendJSONLine(db.walFile, walHead); err != nil { // Remove the WAL created just above: O_EXCL means it is ours, and diff --git a/bundle/direct/dstate/state_test.go b/bundle/direct/dstate/state_test.go index a44afae8fab..23fbc245602 100644 --- a/bundle/direct/dstate/state_test.go +++ b/bundle/direct/dstate/state_test.go @@ -38,15 +38,37 @@ func TestOpenSaveFinalizeRoundTrip(t *testing.T) { mustFinalize(t, &db2) } -func TestFinalizeWithNoEntriesDoesNotWriteStateFile(t *testing.T) { - path := filepath.Join(t.TempDir(), "state.json") - - var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WritingFile, WithRecovery(true), OpenDmsArgs{})) - mustFinalize(t, &db) - - _, err := os.Stat(path) - assert.ErrorIs(t, err, os.ErrNotExist) +func TestFinalizeWithoutResourceWrites(t *testing.T) { + for _, mode := range []Mode{WritingFile, WritingFileWithHistoryEnforced} { + path := filepath.Join(t.TempDir(), "state.json") + var db DeploymentState + require.NoError(t, db.Open(t.Context(), path, ReadingFile, WithRecovery(true), OpenDmsArgs{})) + if mode == WritingFileWithHistoryEnforced { + // A first plan already knows history is enabled, before a state file exists. + db.Data.Features = map[string]struct{}{FeatureDeploymentHistoryBestEffort: {}} + } + require.NoError(t, db.upgradeToWrite(mode)) + mustFinalize(t, &db) + + if mode == WritingFile { + assert.NoFileExists(t, path) + continue + } + // Even an empty deployment must persist its history marker and stable lineage. + require.NoError(t, db.Open(t.Context(), path, ReadingFile, WithRecovery(false), OpenDmsArgs{})) + assert.Equal(t, 1, db.Data.Serial) + assert.NotEmpty(t, db.Data.Lineage) + assert.Contains(t, db.StateFeatures(), FeatureDeploymentHistoryBestEffort) + mustFinalize(t, &db) + + // Turning history off also persists its marker change without resource operations. + require.NoError(t, db.Open(t.Context(), path, WritingFile, WithRecovery(false), OpenDmsArgs{})) + mustFinalize(t, &db) + require.NoError(t, db.Open(t.Context(), path, ReadingFile, WithRecovery(false), OpenDmsArgs{})) + assert.Equal(t, 2, db.Data.Serial) + assert.NotContains(t, db.StateFeatures(), FeatureDeploymentHistoryBestEffort) + mustFinalize(t, &db) + } } func TestDataForFileSerial(t *testing.T) { @@ -423,7 +445,7 @@ func TestOpenFailureLeavesStateClosed(t *testing.T) { } func TestGetSerialWithHistoryUsesLatestCounter(t *testing.T) { - for _, mode := range []Mode{WritingFileWithHistory, WritingFileWithHistoryEnforced} { + for _, mode := range []Mode{ReadingFile, WritingFileWithHistory, WritingFileWithHistoryEnforced} { db := DeploymentState{mode: mode, Data: NewDatabase("", 4), VersionID: 7} assert.Equal(t, 7, db.GetSerial(), "mode=%d: service version is ahead", mode) db.Data.Serial = 9 diff --git a/bundle/phases/bind.go b/bundle/phases/bind.go index d8ff6aeaf3e..7646735f903 100644 --- a/bundle/phases/bind.go +++ b/bundle/phases/bind.go @@ -12,6 +12,7 @@ import ( "github.com/databricks/cli/bundle/config/engine" "github.com/databricks/cli/bundle/deploy/lock" "github.com/databricks/cli/bundle/deployplan" + bundleenv "github.com/databricks/cli/bundle/env" "github.com/databricks/cli/bundle/statemgmt" "github.com/databricks/cli/libs/agent" "github.com/databricks/cli/libs/cmdio" @@ -47,7 +48,11 @@ func Bind(ctx context.Context, b *bundle.Bundle, opts *BindOptions, stateDesc *s return } - if stateDesc.IsDMS() { + var configured *bool + if b.Config.Experimental != nil { + configured = b.Config.Experimental.DeploymentHistory + } + if stateDesc.IsDMS() || bundleenv.RecordsDeploymentHistory(ctx, configured) { logdiag.LogError(ctx, errors.New("bind is not supported for a bundle target that records deployment history")) return } diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index 13c5121f569..78811b0ae7a 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -22,7 +22,6 @@ import ( "github.com/databricks/cli/bundle/statemgmt" "github.com/databricks/cli/libs/agent" "github.com/databricks/cli/libs/cmdio" - "github.com/databricks/cli/libs/dms" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/logdiag" "github.com/databricks/cli/libs/sync" @@ -170,9 +169,9 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand // The version is created only after approval; CompleteVersion is deferred before // lock.Release and no-ops until then. defer func() { - if b.DeploymentBundle.StateDB.IsDeploymentMetadataService() { + if b.DeploymentBundle.StateDB.RecordsHistory() { if _, err := b.DeploymentBundle.StateDB.CompleteVersion(ctx, !logdiag.HasError(ctx)); err != nil { - logdiag.LogError(ctx, err) + log.Warnf(ctx, "Failed to complete deployment history version: %s", err) } } bundle.ApplyContext(ctx, b, lock.Release(lock.GoalDeploy)) @@ -297,7 +296,7 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand // Upgrade from read (opened by process.go, or by Migrate) to write mode. After approval // and the migration commit above, so a declined deploy never opens a WAL it must discard, // and the migration's tf+1 push lands before this advances the WAL header to tf+2. - if err := b.DeploymentBundle.StateDB.UpgradeToWrite(); err != nil { + if err := upgradeStateForWrite(ctx, b); err != nil { logdiag.LogError(ctx, err) return } @@ -320,32 +319,13 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand // Create the deployment now that the plan is approved, so a declined deploy leaves none behind. // A first deploy's id did not exist at plan time - the version and any existing id were stamped // then - so stamp the one just created into the plan the apply reads. - if b.DeploymentBundle.StateDB.IsDeploymentMetadataService() { - firstDeploy := b.DeploymentBundle.StateDB.DeploymentID == "" - createOrUpdateDeployment(ctx, b, dmsDeployment) - if logdiag.HasError(ctx) { - return - } - if firstDeploy { - if err := b.DeploymentBundle.StampDeploymentIdForFirstVersion(b.DeploymentBundle.StateDB.DeploymentID); err != nil { - logdiag.LogError(ctx, err) - return - } - } - - // Only create a version when the plan has at least one operation to record, so it moves - // with serial. Done after the prompt, so a declined deploy never claims a number. - staged, err := stagedOperations(plan) - if err != nil { - logdiag.LogError(ctx, err) - return - } - if len(staged) > 0 { - if err := startVersion(ctx, b, dms.VersionTypeDeploy, staged); err != nil { + if b.DeploymentBundle.StateDB.RecordsHistory() { + if err := prepareDeploymentHistory(ctx, b, plan, dmsDeployment); err != nil { + if b.DeploymentBundle.StateDB.IsDeploymentMetadataService() { logdiag.LogError(ctx, err) return } - logDeploymentVersion(ctx, b) + log.Warnf(ctx, "Could not record deployment history; continuing with file state: %s", err) } } diff --git a/bundle/phases/destroy.go b/bundle/phases/destroy.go index 07bbbbca385..66227fa1c74 100644 --- a/bundle/phases/destroy.go +++ b/bundle/phases/destroy.go @@ -14,9 +14,9 @@ import ( "github.com/databricks/cli/bundle/deploy/files" "github.com/databricks/cli/bundle/deploy/lock" "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/bundle/direct/dstate" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dms" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/logdiag" "github.com/databricks/databricks-sdk-go/apierr" @@ -156,20 +156,22 @@ func destroyCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan) { return } - if b.DeploymentBundle.StateDB.IsDeploymentMetadataService() { + if b.DeploymentBundle.StateDB.RecordsHistory() { // Complete version before deleting remote files; the deployment node is under statePath. completed, err := b.DeploymentBundle.StateDB.CompleteVersion(ctx, true) if err != nil { - logdiag.LogError(ctx, err) - return + log.Warnf(ctx, "Failed to complete deployment history version: %s", err) } // A completed destroy's resources are gone, so its deployment record is deleted too. if completed { deploymentID := b.DeploymentBundle.StateDB.DeploymentID w := b.WorkspaceClient(ctx) if err := deleteDeployment(ctx, w, deploymentID); err != nil { - logdiag.LogError(ctx, fmt.Errorf("failed to delete deployment: %w", err)) - return + if b.DeploymentBundle.StateDB.IsDeploymentMetadataService() { + logdiag.LogError(ctx, fmt.Errorf("failed to delete deployment: %w", err)) + return + } + log.Warnf(ctx, "Could not delete deployment history: %s", err) } } } @@ -293,15 +295,19 @@ func Destroy(ctx context.Context, b *bundle.Bundle) { // destroy records nothing. Deferred before lock.Release to hold the lock; a no-op once // destroyCore has completed the version. defer func() { - if b.DeploymentBundle.StateDB.IsDeploymentMetadataService() { + if b.DeploymentBundle.StateDB.RecordsHistory() { completed, err := b.DeploymentBundle.StateDB.CompleteVersion(ctx, !logdiag.HasError(ctx)) if err != nil { - logdiag.LogError(ctx, err) + log.Warnf(ctx, "Failed to complete deployment history version: %s", err) } else if completed { deploymentID := b.DeploymentBundle.StateDB.DeploymentID w := b.WorkspaceClient(ctx) if err := deleteDeployment(ctx, w, deploymentID); err != nil { - logdiag.LogError(ctx, fmt.Errorf("failed to delete deployment: %w", err)) + if b.DeploymentBundle.StateDB.IsDeploymentMetadataService() { + logdiag.LogError(ctx, fmt.Errorf("failed to delete deployment: %w", err)) + } else { + log.Warnf(ctx, "Could not delete deployment history: %s", err) + } } } } @@ -346,23 +352,26 @@ func Destroy(ctx context.Context, b *bundle.Bundle) { } } - // Upgrade from read (opened by process.go, or by Migrate) to write mode - if err := b.DeploymentBundle.StateDB.UpgradeToWrite(); err != nil { + // History follows the opened state, not configuration. + var err error + if _, history := b.DeploymentBundle.StateDB.StateFeatures()[dstate.FeatureDeploymentHistoryBestEffort]; history { + err = b.DeploymentBundle.StateDB.UpgradeToWriteWithHistory() + } else { + err = b.DeploymentBundle.StateDB.UpgradeToWrite() + } + if err != nil { logdiag.LogError(ctx, err) return } - // Start the version for this destroy, now that it is approved. Everything else recording - // does follows from the buffer this opens. - if b.DeploymentBundle.StateDB.IsDeploymentMetadataService() { - staged, err := stagedOperations(plan) - if err != nil { - logdiag.LogError(ctx, err) - return - } - if err := startVersion(ctx, b, dms.VersionTypeDestroy, staged); err != nil { - logdiag.LogError(ctx, err) - return + // Record the approved destroy; file-backed history errors only warn. + if b.DeploymentBundle.StateDB.RecordsHistory() { + if err := prepareDestroyHistory(ctx, b, plan); err != nil { + if b.DeploymentBundle.StateDB.IsDeploymentMetadataService() { + logdiag.LogError(ctx, err) + return + } + log.Warnf(ctx, "Could not record deployment history; continuing with file state: %s", err) } } destroyCore(ctx, b, plan) diff --git a/bundle/phases/dms.go b/bundle/phases/dms.go index e2d7ea958c1..1bf195a6034 100644 --- a/bundle/phases/dms.go +++ b/bundle/phases/dms.go @@ -12,11 +12,11 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/deployplan" + bundleenv "github.com/databricks/cli/bundle/env" "github.com/databricks/cli/internal/build" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/dms" "github.com/databricks/cli/libs/log" - "github.com/databricks/cli/libs/logdiag" "github.com/databricks/cli/libs/retry" "github.com/databricks/cli/libs/workspaceurls" "github.com/databricks/databricks-sdk-go" @@ -73,13 +73,12 @@ func actionToSDK(a deployplan.ActionType) (bundledeployments.OperationActionType // run changed. current is the record the service holds (nil before the first recorded deploy), // diffed to mask the update down to what changed. Runs after approval, so a declined deploy leaves // no deployment behind; a first deploy's new id is then stamped into the plan (StampDeploymentID). -func createOrUpdateDeployment(ctx context.Context, b *bundle.Bundle, current *bundledeployments.Deployment) { +func createOrUpdateDeployment(ctx context.Context, b *bundle.Bundle, current *bundledeployments.Deployment) error { db := &b.DeploymentBundle w := b.WorkspaceClient(ctx) metadata, err := deploymentMetadata(b) if err != nil { - logdiag.LogError(ctx, fmt.Errorf("failed to compute deployment metadata: %w", err)) - return + return fmt.Errorf("failed to compute deployment metadata: %w", err) } deploymentID := db.StateDB.DeploymentID if deploymentID == "" { @@ -87,14 +86,12 @@ func createOrUpdateDeployment(ctx context.Context, b *bundle.Bundle, current *bu dep.InitialParentPath = b.Config.Workspace.StatePath created, err := w.BundleDeployments.CreateDeployment(ctx, bundledeployments.CreateDeploymentRequest{Deployment: dep}) if err != nil { - logdiag.LogError(ctx, fmt.Errorf("failed to create deployment: %w", err)) - return + return fmt.Errorf("failed to create deployment: %w", err) } // The id is the workspace node the server makes under the parent path. deploymentID, err = dms.DeploymentIDFromName(created.Name) if err != nil { - logdiag.LogError(ctx, fmt.Errorf("failed to create deployment: %w", err)) - return + return fmt.Errorf("failed to create deployment: %w", err) } db.StateDB.DeploymentID = deploymentID } else if mask := metadata.StaleFields(current); mask != "" { @@ -106,8 +103,7 @@ func createOrUpdateDeployment(ctx context.Context, b *bundle.Bundle, current *bu }) }) if err != nil { - logdiag.LogError(ctx, fmt.Errorf("failed to update deployment: %w", err)) - return + return fmt.Errorf("failed to update deployment: %w", err) } } @@ -115,6 +111,7 @@ func createOrUpdateDeployment(ctx context.Context, b *bundle.Bundle, current *bu bundle.ApplyFuncContext(ctx, b, func(_ context.Context, b *bundle.Bundle) { b.Config.Bundle.Deployment.DeploymentID = deploymentID }) + return nil } // deleteDeployment deletes the deployment record. A retry that finds it missing succeeds: the @@ -132,12 +129,67 @@ func deleteDeployment(ctx context.Context, w *databricks.WorkspaceClient, deploy return err } -// startVersion claims the version the run settled on and opens the buffer that records -// each state write under it. Called after approval, so a declined deploy never claims a number. +// upgradeStateForWrite delays the history WAL until the operation is approved. +func upgradeStateForWrite(ctx context.Context, b *bundle.Bundle) error { + db := &b.DeploymentBundle.StateDB + var configured *bool + if b.Config.Experimental != nil { + configured = b.Config.Experimental.DeploymentHistory + } + if !db.IsDeploymentMetadataService() && bundleenv.RecordsDeploymentHistory(ctx, configured) { + return db.UpgradeToWriteWithHistory() + } + return db.UpgradeToWrite() +} + +// prepareDeploymentHistory creates history metadata and a version after approval. +func prepareDeploymentHistory(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, current *bundledeployments.Deployment) error { + firstDeploy := b.DeploymentBundle.StateDB.DeploymentID == "" + if err := createOrUpdateDeployment(ctx, b, current); err != nil { + return err + } + if firstDeploy { + if err := b.DeploymentBundle.StampDeploymentIdForFirstVersion(b.DeploymentBundle.StateDB.DeploymentID); err != nil { + return err + } + } + staged, err := stagedOperations(plan) + if err != nil { + return err + } + if len(staged) == 0 { + return nil + } + if err := startVersion(ctx, b, dms.VersionTypeDeploy, staged); err != nil { + return err + } + logDeploymentVersion(ctx, b) + return nil +} + +// prepareDestroyHistory records the approved destroy without changing its state source. +func prepareDestroyHistory(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan) error { + if !b.DeploymentBundle.StateDB.IsDeploymentMetadataService() && b.DeploymentBundle.StateDB.DeploymentID == "" { + if err := createOrUpdateDeployment(ctx, b, nil); err != nil { + return err + } + } + staged, err := stagedOperations(plan) + if err != nil { + return err + } + return startVersion(ctx, b, dms.VersionTypeDestroy, staged) +} + +// startVersion opens a history version after approval. File-backed history skips empty versions; +// DMS-only destroy retains its existing behavior. A declined deploy never claims a version. // A no-op when the bundle does not record deployment history. func startVersion(ctx context.Context, b *bundle.Bundle, versionType dms.VersionType, staged []bundledeployments.StagedOperation) error { db := &b.DeploymentBundle - if !db.StateDB.IsDeploymentMetadataService() { + if !db.StateDB.RecordsHistory() { + return nil + } + if len(staged) == 0 && !db.StateDB.IsDeploymentMetadataService() { return nil } w := b.WorkspaceClient(ctx) @@ -146,7 +198,7 @@ func startVersion(ctx context.Context, b *bundle.Bundle, versionType dms.Version // InitializeOperationBuffer moves the anchor below, so callers afterwards read the created // version straight off the state. previous := db.StateDB.VersionID - versionID := previous + 1 + versionID := max(previous, db.StateDB.Data.Serial) + 1 previousVersionID := "" if previous > 0 { previousVersionID = strconv.Itoa(previous) diff --git a/bundle/phases/telemetry.go b/bundle/phases/telemetry.go index 99464f3ef24..102a004aa1a 100644 --- a/bundle/phases/telemetry.go +++ b/bundle/phases/telemetry.go @@ -198,7 +198,7 @@ func aiRuntimeTaskMetrics(jobs map[string]*resources.Job) (present, scheduled, m // LogDeployTelemetry logs a telemetry event for a bundle deploy command. func LogDeployTelemetry(ctx context.Context, b *bundle.Bundle, errMsg string) { errMsg = telemetry.ScrubErrorMessage(errMsg) - b.Metrics.SetBoolValue(metrics.DeploymentHistoryEnabled, b.DeploymentBundle.StateDB.IsDeploymentMetadataService()) + b.Metrics.SetBoolValue(metrics.DeploymentHistoryEnabled, b.DeploymentBundle.StateDB.RecordsHistory()) resourcesCount := int64(0) _, err := dyn.MapByPattern(b.Config.Value(), dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey()), func(p dyn.Path, v dyn.Value) (dyn.Value, error) { diff --git a/bundle/statemgmt/state_pull.go b/bundle/statemgmt/state_pull.go index 616cd8af197..328cfe8761f 100644 --- a/bundle/statemgmt/state_pull.go +++ b/bundle/statemgmt/state_pull.go @@ -77,6 +77,12 @@ func (s *StateDesc) IsDMS() bool { return ok } +// RecordsHistory includes both DMS-authoritative state and file-backed history. +func (s *StateDesc) RecordsHistory() bool { + _, bestEffort := s.Features["deployment_history_best_effort"] + return bestEffort || s.IsDMS() +} + func localRead(ctx context.Context, fullPath string, engine engine.EngineType) *StateDesc { content, err := os.ReadFile(fullPath) if err != nil { diff --git a/cmd/bundle/utils/destroy.go b/cmd/bundle/utils/destroy.go index 7df0a78c328..54c5911e7fa 100644 --- a/cmd/bundle/utils/destroy.go +++ b/cmd/bundle/utils/destroy.go @@ -43,9 +43,9 @@ func CommandBundleDestroy(cmd *cobra.Command, args []string, autoApprove, forceD } }, // Skip context initialization if already initialized by parent command - SkipInitContext: skipInitContext, - AlwaysPull: true, - SkipEnforcingDeploymentHistorySetting: true, + SkipInitContext: skipInitContext, + AlwaysPull: true, + Destroy: true, PostStateFunc: func(ctx context.Context, b *bundle.Bundle, stateDesc *statemgmt.StateDesc) error { phases.Destroy(ctx, b) if logdiag.HasError(ctx) { diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index 8fa68f32f5e..4a5e666dabb 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -96,8 +96,8 @@ type ProcessOptions struct { // (after state is opened and IDs loaded, before deferred Finalize). PostStateFunc func(ctx context.Context, b *bundle.Bundle, stateDesc *statemgmt.StateDesc) error - // If true, deployment history configuration is ignored after state is resolved. - SkipEnforcingDeploymentHistorySetting bool + // Destroy selects history from loaded state instead of configuration. + Destroy bool // Indicate whether the bundle operation originates from the pipelines CLI IsPipelinesCLI bool @@ -249,9 +249,6 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle } ctx = WithLineage(ctx, stateDesc) cmd.SetContext(ctx) - if stateDesc.Engine.IsDirect() { - resolveDeploymentHistory(ctx, b, stateDesc) - } // Record the engine the resolved state uses now, so deploy telemetry reports // it even when the deploy fails or is cancelled before deployCore runs. @@ -365,14 +362,44 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle } } - if stateDesc.Engine.IsDirect() && !opts.SkipEnforcingDeploymentHistorySetting { + if stateDesc.Engine.IsDirect() && !opts.Destroy { if err := enforceDeploymentHistorySetting(ctx, b, stateDesc, opts.Deploy || opts.PreDeployChecks); err != nil { logdiag.LogError(ctx, err) return b, stateDesc, root.ErrAlreadyPrinted } } - if needDirectState && stateDesc.IsDMS() { + stateMode := dstate.WritingFile + switch { + case stateDesc.IsDMS(): + // DMS-only state must keep its service-backed source. + stateMode = dstate.WritingDMS + case opts.Destroy: + // Destroy follows loaded state even when the setting has changed. + if b.DeploymentBundle.StateDB.RecordsHistory() { + stateMode = dstate.WritingFileWithHistoryEnforced + } + case configuresDeploymentHistory(ctx, b): + // Deploy and plan follow the current configuration. + stateMode = dstate.WritingFileWithHistoryEnforced + if opts.Deploy || opts.PreDeployChecks { + // Retain only this history feature for plan/deploy; overwrite all other flags. + b.DeploymentBundle.StateDB.Data.Features = map[string]struct{}{dstate.FeatureDeploymentHistoryBestEffort: {}} + } + } + + if needDirectState && stateMode == dstate.WritingFileWithHistoryEnforced { + if !cmdctx.HasWorkspaceClient(ctx) { + ctx = cmdctx.SetWorkspaceClient(ctx, b.WorkspaceClient(ctx)) + } + var err error + dmsDeployment, err = prepareFileHistory(ctx, b) + if err != nil { + log.Warnf(ctx, "Could not read deployment history; continuing with file state: %s", err) + } + } + + if needDirectState && stateMode == dstate.WritingDMS { var err error dmsDeploymentID, dmsDeployment, err = fetchDeploymentFromStatePath(ctx, b.WorkspaceClient(ctx), b.Config.Workspace.StatePath) if err != nil { @@ -638,11 +665,19 @@ func parseLastVersionID(dmsDeployment *bundledeployments.Deployment) (int, error // read from the deployment metadata service instead. func OpenDirectStateForRead(ctx context.Context, b *bundle.Bundle, stateDesc *statemgmt.StateDesc) error { _, localPath := b.StateFilenameDirect(ctx) - if !resolveDeploymentHistory(ctx, b, stateDesc) { + if !stateDesc.IsDMS() { if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.ReadingFile, dstate.WithRecovery(true), dstate.OpenDmsArgs{}); err != nil { return err } - return enforceDeploymentHistorySetting(ctx, b, stateDesc, false) + if err := enforceDeploymentHistorySetting(ctx, b, stateDesc, false); err != nil { + return err + } + if configuresDeploymentHistory(ctx, b) { + if _, err := prepareFileHistory(ctx, b); err != nil { + log.Warnf(ctx, "Could not read deployment history; continuing with file state: %s", err) + } + } + return nil } dmsDeploymentID, dmsDeployment, err := fetchDeploymentFromStatePath(ctx, b.WorkspaceClient(ctx), b.Config.Workspace.StatePath) @@ -663,25 +698,16 @@ func OpenDirectStateForRead(ctx context.Context, b *bundle.Bundle, stateDesc *st return enforceDeploymentHistorySetting(ctx, b, stateDesc, false) } -func resolveDeploymentHistory(ctx context.Context, b *bundle.Bundle, stateDesc *statemgmt.StateDesc) bool { - configured := configuresDeploymentHistory(ctx, b) - if stateDesc.SourcePath == "" { - if configured { - stateDesc.Features = map[string]struct{}{dstate.FeatureDeploymentHistory: {}} - } - return configured - } - - return stateDesc.IsDMS() -} - func enforceDeploymentHistorySetting(ctx context.Context, b *bundle.Bundle, stateDesc *statemgmt.StateDesc, requireMatch bool) error { if stateDesc.SourcePath == "" { return nil } configured := configuresDeploymentHistory(ctx, b) - recorded := stateDesc.IsDMS() - if configured == recorded { + recorded := stateDesc.RecordsHistory() + if b.DeploymentBundle.StateDB.IsOpen() { + recorded = b.DeploymentBundle.StateDB.RecordsHistory() + } + if configured == recorded || (!configured && !stateDesc.IsDMS()) { return nil } if requireMatch { @@ -698,6 +724,29 @@ Run "databricks bundle destroy" first, then deploy again with deployment history return nil } +// prepareFileHistory reads version metadata, never resource state, from DMS. +func prepareFileHistory(ctx context.Context, b *bundle.Bundle) (*bundledeployments.Deployment, error) { + id, deployment, err := fetchDeploymentFromStatePath(ctx, b.WorkspaceClient(ctx), b.Config.Workspace.StatePath) + if err != nil { + return nil, err + } + lastVersion, err := parseLastVersionID(deployment) + if err != nil { + return nil, err + } + db := &b.DeploymentBundle.StateDB + db.DeploymentID, db.VersionID = id, lastVersion + bundle.ApplyFuncContext(ctx, b, func(_ context.Context, b *bundle.Bundle) { + b.Config.Bundle.Deployment.DeploymentID = id + b.Config.Bundle.Deployment.LatestVersionID = lastVersion + }) + bundle.ApplyContext(ctx, b, metadata.AnnotateDeploymentVersion(max(db.Data.Serial, lastVersion)+1)) + if id != "" { + bundle.ApplyContext(ctx, b, metadata.AnnotateDeployment(id)) + } + return deployment, nil +} + func configuresDeploymentHistory(ctx context.Context, b *bundle.Bundle) bool { var configured *bool if b.Config.Experimental != nil { diff --git a/cmd/bundle/utils/process_test.go b/cmd/bundle/utils/process_test.go index b216f1102d2..21910039344 100644 --- a/cmd/bundle/utils/process_test.go +++ b/cmd/bundle/utils/process_test.go @@ -3,31 +3,11 @@ package utils import ( "testing" - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/bundle/direct/dstate" - "github.com/databricks/cli/bundle/statemgmt" "github.com/databricks/databricks-sdk-go/service/bundledeployments" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -func TestResolveDeploymentHistory(t *testing.T) { - b := &bundle.Bundle{Config: config.Root{ - Experimental: &config.Experimental{DeploymentHistory: new(true)}, - }} - - newState := &statemgmt.StateDesc{} - assert.True(t, resolveDeploymentHistory(t.Context(), b, newState)) - assert.True(t, newState.IsDMS()) - assert.False(t, resolveDeploymentHistory(t.Context(), b, &statemgmt.StateDesc{SourcePath: "resources.json"})) - b.Config.Experimental.DeploymentHistory = new(false) - assert.True(t, resolveDeploymentHistory(t.Context(), b, &statemgmt.StateDesc{ - SourcePath: "resources.json", - Features: map[string]struct{}{dstate.FeatureDeploymentHistory: {}}, - })) -} - func TestIsNewerVersion(t *testing.T) { tests := []struct { name string From db9b734dd636ec84455bd41f6091fefd9fb37201 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 26 Oct 2026 16:00:53 +0000 Subject: [PATCH 2/2] Preserve file serial when history markers are unchanged --- .../dms/deployment-metadata-change/output.txt | 2 +- acceptance/bundle/dms/failed-delete/output.txt | 2 +- .../dms/failed-update-permissions/output.txt | 2 +- .../bundle/dms/multiple-resources/output.txt | 4 ++-- acceptance/bundle/dms/no-resources/output.txt | 4 ++-- bundle/direct/dstate/state.go | 15 +++++++++++---- bundle/direct/dstate/state_test.go | 17 +++++++++++++++++ 7 files changed, 35 insertions(+), 11 deletions(-) diff --git a/acceptance/bundle/dms/deployment-metadata-change/output.txt b/acceptance/bundle/dms/deployment-metadata-change/output.txt index ee12a197034..1f14fe5797e 100644 --- a/acceptance/bundle/dms/deployment-metadata-change/output.txt +++ b/acceptance/bundle/dms/deployment-metadata-change/output.txt @@ -126,7 +126,7 @@ A common practice is to use a username or principal name in this path, i.e. use "plan_version": [PLAN_VERSION], "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", - "serial": 2, + "serial": 1, "features": { "deployment_history_best_effort": {} } diff --git a/acceptance/bundle/dms/failed-delete/output.txt b/acceptance/bundle/dms/failed-delete/output.txt index db348b73d61..9cd4b49e3dc 100644 --- a/acceptance/bundle/dms/failed-delete/output.txt +++ b/acceptance/bundle/dms/failed-delete/output.txt @@ -201,7 +201,7 @@ API message: Fault injected by test. "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", - "serial": 2, + "serial": 1, "features": { "deployment_history_best_effort": {} }, diff --git a/acceptance/bundle/dms/failed-update-permissions/output.txt b/acceptance/bundle/dms/failed-update-permissions/output.txt index 99b4f03417c..7ab42c6e531 100644 --- a/acceptance/bundle/dms/failed-update-permissions/output.txt +++ b/acceptance/bundle/dms/failed-update-permissions/output.txt @@ -139,7 +139,7 @@ Files: 1 uploaded, 0 deleted "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", - "serial": 2, + "serial": 1, "features": { "deployment_history_best_effort": {} }, diff --git a/acceptance/bundle/dms/multiple-resources/output.txt b/acceptance/bundle/dms/multiple-resources/output.txt index 2979ca25346..e045271320b 100644 --- a/acceptance/bundle/dms/multiple-resources/output.txt +++ b/acceptance/bundle/dms/multiple-resources/output.txt @@ -180,7 +180,7 @@ Resources: 0 created, 0 changed, 0 deleted, 5 unchanged "plan_version": [PLAN_VERSION], "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", - "serial": 2, + "serial": 1, "features": { "deployment_history_best_effort": {} } @@ -225,7 +225,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 5 unchanged "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", - "serial": 2, + "serial": 1, "features": { "deployment_history_best_effort": {} }, diff --git a/acceptance/bundle/dms/no-resources/output.txt b/acceptance/bundle/dms/no-resources/output.txt index 6c44aa176ab..0899a983e1c 100644 --- a/acceptance/bundle/dms/no-resources/output.txt +++ b/acceptance/bundle/dms/no-resources/output.txt @@ -91,7 +91,7 @@ Resources: 0 created, 0 changed, 0 deleted, 0 unchanged "plan_version": [PLAN_VERSION], "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", - "serial": 2, + "serial": 1, "features": { "deployment_history_best_effort": {} } @@ -110,7 +110,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 0 unchanged "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", - "serial": 2, + "serial": 1, "features": { "deployment_history_best_effort": {} }, diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index a8e2a441dda..0b612f32358 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -93,6 +93,9 @@ type DeploymentState struct { mu sync.Mutex walFile *os.File + // persistedFeatures tracks disk markers separately from configuration changes in Data. + persistedFeatures map[string]struct{} + // mode controls where resource state is read and written, whether DMS // history is recorded, and whether DMS errors fail the deployment. mode Mode @@ -560,6 +563,7 @@ func (db *DeploymentState) reset() { db.Path = "" db.Data = Database{} db.stateIDs = nil + db.persistedFeatures = nil // Telemetry and destroy still need the mode and IDs after version completion. // Clear the recording buffer once CompleteVersion has copied it. if db.versionCompleted { @@ -593,6 +597,7 @@ func (db *DeploymentState) unlockedOpen(ctx context.Context, path string, mode M return fmt.Errorf("migrating state %s: %w", path, err) } + db.persistedFeatures = maps.Clone(db.Data.Features) db.stateIDs = make(map[string]string) for key, entry := range db.Data.State { db.stateIDs[key] = entry.ID @@ -812,9 +817,9 @@ func (db *DeploymentState) mergeWalIntoState(ctx context.Context) (bool, error) } hasEntries := lineNumber > 1 - // Persist enforced-history headers even without resource writes. Configured features - // may already match in memory while the file still carries different markers. - if lineNumber == 1 && (db.mode == WritingFileWithHistoryEnforced || !maps.Equal(db.Data.Features, newFeatures)) { + // Persist marker changes even without resource writes. Compare against disk, + // because configuration may already have replaced the in-memory features. + if lineNumber == 1 && !maps.Equal(db.persistedFeatures, newFeatures) { hasEntries = true } @@ -1080,7 +1085,8 @@ func (db *DeploymentState) ExportState(ctx context.Context) resourcestate.Export // the WAL. A torn write would therefore leave a state file that Open rejects // next to an intact WAL it never reads. func (db *DeploymentState) unlockedSave() error { - data, err := json.MarshalIndent(db.dataForFile(), "", " ") + fileData := db.dataForFile() + data, err := json.MarshalIndent(fileData, "", " ") if err != nil { return err } @@ -1089,6 +1095,7 @@ func (db *DeploymentState) unlockedSave() error { return fmt.Errorf("failed to save resources state to %#v: %w", db.Path, err) } + db.persistedFeatures = maps.Clone(fileData.Features) return nil } diff --git a/bundle/direct/dstate/state_test.go b/bundle/direct/dstate/state_test.go index 23fbc245602..8f12b608e71 100644 --- a/bundle/direct/dstate/state_test.go +++ b/bundle/direct/dstate/state_test.go @@ -61,6 +61,13 @@ func TestFinalizeWithoutResourceWrites(t *testing.T) { assert.Contains(t, db.StateFeatures(), FeatureDeploymentHistoryBestEffort) mustFinalize(t, &db) + // Keeping history enabled without changes must not advance the file serial. + require.NoError(t, db.Open(t.Context(), path, WritingFileWithHistoryEnforced, WithRecovery(false), OpenDmsArgs{})) + mustFinalize(t, &db) + require.NoError(t, db.Open(t.Context(), path, ReadingFile, WithRecovery(false), OpenDmsArgs{})) + assert.Equal(t, 1, db.Data.Serial) + mustFinalize(t, &db) + // Turning history off also persists its marker change without resource operations. require.NoError(t, db.Open(t.Context(), path, WritingFile, WithRecovery(false), OpenDmsArgs{})) mustFinalize(t, &db) @@ -68,6 +75,16 @@ func TestFinalizeWithoutResourceWrites(t *testing.T) { assert.Equal(t, 2, db.Data.Serial) assert.NotContains(t, db.StateFeatures(), FeatureDeploymentHistoryBestEffort) mustFinalize(t, &db) + + // Enabling history on existing state persists the marker, even without resource writes. + require.NoError(t, db.Open(t.Context(), path, ReadingFile, WithRecovery(false), OpenDmsArgs{})) + db.Data.Features = map[string]struct{}{FeatureDeploymentHistoryBestEffort: {}} + require.NoError(t, db.upgradeToWrite(WritingFileWithHistoryEnforced)) + mustFinalize(t, &db) + require.NoError(t, db.Open(t.Context(), path, ReadingFile, WithRecovery(false), OpenDmsArgs{})) + assert.Equal(t, 3, db.Data.Serial) + assert.Contains(t, db.StateFeatures(), FeatureDeploymentHistoryBestEffort) + mustFinalize(t, &db) } }