Avoid syncing cluster policy values into shared bundle config

.nextchanges/bundles/remote-sync-cluster-policies.md

@@ -0,0 +1 @@
+* Fix `bundle config-remote-sync` copying cluster policy values into shared configuration and breaking deployment to another target. ([#7170](https://github.com/databricks/cli/pull/7170))

acceptance/bundle/config-remote-sync/cluster_policy_cross_target/databricks.yml.tmpl

@@ -0,0 +1,52 @@
+bundle:
+  name: test-bundle-$UNIQUE_NAME
+
+workspace:
+  root_path: ~/.bundle/$UNIQUE_NAME/${bundle.target}
+
+variables:
+  environment:
+    default: dev
+
+resources:
+  cluster_policies:
+    policy:
+      name: test-policy-$UNIQUE_NAME-${bundle.target}
+      definition:
+        custom_tags.workspace:
+          type: fixed
+          value: ${var.environment}
+        spark_env_vars.ENVIRONMENT:
+          type: fixed
+          value: ${var.environment}
+        workload_type.clients.jobs:
+          type: fixed
+          value: true
+        workload_type.clients.notebooks:
+          type: fixed
+          value: false
+  jobs:
+    policy_sync:
+      name: test-job-$UNIQUE_NAME-${bundle.target}
+      description: Before UI edit
+      job_clusters:
+        - job_cluster_key: main
+          new_cluster:
+            policy_id: ${resources.cluster_policies.policy.id}
+            spark_version: $DEFAULT_SPARK_VERSION
+            node_type_id: $NODE_TYPE_ID
+            num_workers: 1
+            custom_tags:
+              owner: bundle-author
+      tasks:
+        - task_key: main
+          job_cluster_key: main
+          spark_python_task:
+            python_file: ./hello.py
+
+targets:
+  dev:
+    default: true
+  prod:
+    variables:
+      environment: prod

acceptance/bundle/config-remote-sync/cluster_policy_cross_target/hello.py

@@ -0,0 +1 @@
+print("hello")

acceptance/bundle/config-remote-sync/cluster_policy_cross_target/out.test.toml

@@ -0,0 +1,3 @@
+Cloud = true
+GOOS.windows = false
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/config-remote-sync/cluster_policy_cross_target/output.txt

@@ -0,0 +1,87 @@
+
+=== Deploy with the dev policy
+>>> [CLI] bundle deploy -t dev
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/dev/files...
+Created cluster_policies.policy
+Created jobs.policy_sync
+Files: 3 uploaded, 0 deleted
+Resources: 2 created, 0 changed, 0 deleted, 0 unchanged
+
+=== Sync the UI edit into shared YAML
+>>> [CLI] bundle config-remote-sync -t dev --save
+Detected changes in 1 resource(s):
+
+Resource: resources.jobs.policy_sync
+  description: replace
+
+
+
+>>> diff.py databricks.yml.backup databricks.yml
+--- databricks.yml.backup
++++ databricks.yml
+@@ -29,5 +29,5 @@
+     policy_sync:
+       name: test-job-[UNIQUE_NAME]-${bundle.target}
+-      description: Before UI edit
++      description: Edited in the UI
+       job_clusters:
+         - job_cluster_key: main
+
+=== Deploy shared YAML with the prod policy
+>>> [CLI] bundle deploy -t prod
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/prod/files...
+Created cluster_policies.policy
+Created jobs.policy_sync
+Files: 4 uploaded, 0 deleted
+Resources: 2 created, 0 changed, 0 deleted, 0 unchanged
+
+=== Add a user tag beside the policy tag in the UI
+>>> [CLI] bundle config-remote-sync -t dev --save
+Detected changes in 1 resource(s):
+
+Resource: resources.jobs.policy_sync
+  job_clusters[job_cluster_key='main'].new_cluster.custom_tags['UserTag']: add
+
+
+
+>>> diff.py databricks.yml.backup databricks.yml
+--- databricks.yml.backup
++++ databricks.yml
+@@ -39,4 +39,5 @@
+             custom_tags:
+               owner: bundle-author
++              UserTag: edited-in-ui
+       tasks:
+         - task_key: main
+
+=== Deploy the user tag with the prod policy
+>>> [CLI] bundle deploy -t prod
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/prod/files...
+Updated jobs.policy_sync
+Files: 2 uploaded, 0 deleted
+Resources: 0 created, 1 changed, 0 deleted, 1 unchanged
+
+>>> [CLI] jobs get [POLICY_SYNC_ID_2] -o json
+{
+  "UserTag": "edited-in-ui",
+  "owner": "bundle-author",
+  "workspace": "prod"
+}
+
+>>> [CLI] bundle destroy -t prod --auto-approve
+The following resources will be deleted:
+  delete resources.cluster_policies.policy
+  delete resources.jobs.policy_sync
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/prod
+
+Destroy: 2 deleted
+
+>>> [CLI] bundle destroy -t dev --auto-approve
+The following resources will be deleted:
+  delete resources.cluster_policies.policy
+  delete resources.jobs.policy_sync
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/dev
+
+Destroy: 2 deleted

acceptance/bundle/config-remote-sync/cluster_policy_cross_target/script

@@ -0,0 +1,37 @@
+envsubst < databricks.yml.tmpl > databricks.yml
+
+cleanup() {
+    trace $CLI bundle destroy -t prod --auto-approve
+    trace $CLI bundle destroy -t dev --auto-approve
+}
+trap cleanup EXIT
+
+title "Deploy with the dev policy"
+trace $CLI bundle deploy -t dev
+job_id=$(read_id.py -t dev policy_sync)
+
+edit_resource.py jobs "$job_id" <<'EOF'
+r["description"] = "Edited in the UI"
+EOF
+
+title "Sync the UI edit into shared YAML"
+cp databricks.yml databricks.yml.backup
+trace $CLI bundle config-remote-sync -t dev --save
+trace diff.py databricks.yml.backup databricks.yml
+
+title "Deploy shared YAML with the prod policy"
+trace $CLI bundle deploy -t prod
+
+title "Add a user tag beside the policy tag in the UI"
+edit_resource.py jobs "$job_id" <<'EOF'
+r["job_clusters"][0]["new_cluster"]["custom_tags"]["UserTag"] = "edited-in-ui"
+EOF
+
+cp databricks.yml databricks.yml.backup
+trace $CLI bundle config-remote-sync -t dev --save
+trace diff.py databricks.yml.backup databricks.yml
+
+title "Deploy the user tag with the prod policy"
+trace $CLI bundle deploy -t prod
+prod_job_id=$(read_id.py -t prod policy_sync)
+trace $CLI jobs get "$prod_job_id" -o json | jq '.settings.job_clusters[0].new_cluster.custom_tags'

acceptance/bundle/config-remote-sync/cluster_policy_cross_target/test.toml

@@ -0,0 +1,3 @@
+Cloud = true
+RecordRequests = false
+Ignore = [".databricks", "databricks.yml", "databricks.yml.backup"]

acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/output.txt

@@ -10,21 +10,53 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged
 Detected changes in 1 resource(s):
 
 Resource: resources.jobs.test_job
+  tasks[task_key='test'].new_cluster.custom_tags: add
   tasks[task_key='test'].new_cluster.spark_conf: add
 
 
 
 >>> diff.py databricks.yml.backup databricks.yml
 --- databricks.yml.backup
 +++ databricks.yml
-@@ -24,4 +24,6 @@
+@@ -24,4 +24,8 @@
              node_type_id: [NODE_TYPE_ID]
              num_workers: 1
++            custom_tags:
++              UserTag: edited-in-ui
 +            spark_conf:
 +              spark.sql.shuffle.partitions: "7"
            spark_python_task:
              python_file: ./hello.py
 
+=== Add a task with user and policy cluster fields in the UI
+>>> [CLI] bundle config-remote-sync --save
+Detected changes in 1 resource(s):
+
+Resource: resources.jobs.test_job
+  tasks[task_key='added_in_ui']: add
+  tasks[task_key='test'].new_cluster.custom_tags: replace
+
+
+
+>>> diff.py databricks.yml.backup databricks.yml
+--- databricks.yml.backup
++++ databricks.yml
+@@ -30,2 +30,14 @@
+           spark_python_task:
+             python_file: ./hello.py
++        - new_cluster:
++            custom_tags:
++              UserTag: new-task
++            node_type_id: [NODE_TYPE_ID]
++            num_workers: 1
++            policy_id: ${resources.cluster_policies.policy.id}
++            spark_conf:
++              spark.sql.shuffle.partitions: "7"
++            spark_version: [DEFAULT_SPARK_VERSION]
++          spark_python_task:
++            python_file: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files/hello.py
++          task_key: added_in_ui
+
 >>> [CLI] bundle destroy --auto-approve
 The following resources will be deleted:
   delete resources.cluster_policies.policy

acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/script

@@ -1,4 +1,5 @@
 envsubst < databricks.yml.tmpl > databricks.yml
+add_repl "$DEFAULT_SPARK_VERSION" DEFAULT_SPARK_VERSION
 
 cleanup() {
     trace $CLI bundle destroy --auto-approve
@@ -10,6 +11,22 @@ job_id=$(read_id.py test_job)
 
 edit_resource.py jobs "$job_id" <<'EOF'
 r["tasks"][0]["new_cluster"]["spark_conf"] = {"spark.sql.shuffle.partitions": "7"}
+r["tasks"][0]["new_cluster"]["custom_tags"]["UserTag"] = "edited-in-ui"
+EOF
+
+cp databricks.yml databricks.yml.backup
+trace $CLI bundle config-remote-sync --save
+trace diff.py databricks.yml.backup databricks.yml
+rm databricks.yml.backup
+
+title "Add a task with user and policy cluster fields in the UI"
+edit_resource.py jobs "$job_id" <<'EOF'
+from copy import deepcopy
+
+task = deepcopy(r["tasks"][0])
+task["task_key"] = "added_in_ui"
+task["new_cluster"]["custom_tags"]["UserTag"] = "new-task"
+r["tasks"].append(task)
 EOF
 
 cp databricks.yml databricks.yml.backup

acceptance/bundle/config-remote-sync/cluster_policy_replaces_array/databricks.yml.tmpl

@@ -0,0 +1,24 @@
+bundle:
+  name: test-bundle-$UNIQUE_NAME
+
+workspace:
+  root_path: ~/.bundle/$UNIQUE_NAME
+
+resources:
+  jobs:
+    test_job:
+      name: test-job-$UNIQUE_NAME
+      tasks:
+        - task_key: main
+          new_cluster:
+            policy_id: $POLICY_ID
+            spark_version: $DEFAULT_SPARK_VERSION
+            node_type_id: $NODE_TYPE_ID
+            num_workers: 1
+            init_scripts:
+              - workspace:
+                  destination: /Workspace/init-a.sh
+              - workspace:
+                  destination: /Workspace/init-b.sh
+          spark_python_task:
+            python_file: ./hello.py

acceptance/bundle/config-remote-sync/cluster_policy_replaces_array/hello.py

@@ -0,0 +1 @@
+print("hello")

acceptance/bundle/config-remote-sync/cluster_policy_replaces_array/out.test.toml

@@ -0,0 +1,3 @@
+Cloud = false
+GOOS.windows = false
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/config-remote-sync/cluster_policy_replaces_array/output.txt

@@ -0,0 +1,49 @@
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files...
+Created jobs.test_job
+Files: 5 uploaded, 0 deleted
+Resources: 1 created, 0 changed, 0 deleted, 0 unchanged
+
+=== Replace authored scripts with a script from an updated policy
+>>> [CLI] cluster-policies edit --json {
+  "policy_id": "[POLICY_ID]",
+  "name": "test-policy-[UNIQUE_NAME]",
+  "definition": "{\"init_scripts.0.volumes.destination\":{\"type\":\"fixed\",\"value\":\"/Volumes/catalog/schema/volume/policy.sh\"}}"
+}
+
+>>> [CLI] bundle config-remote-sync --save
+Detected changes in 1 resource(s):
+
+Resource: resources.jobs.test_job
+  tasks[task_key='main'].new_cluster.init_scripts: remove
+
+
+
+>>> diff.py databricks.yml.backup databricks.yml
+--- databricks.yml.backup
++++ databricks.yml
+@@ -16,9 +16,4 @@
+             node_type_id: [NODE_TYPE_ID]
+             num_workers: 1
+-            init_scripts:
+-              - workspace:
+-                  destination: /Workspace/init-a.sh
+-              - workspace:
+-                  destination: /Workspace/init-b.sh
+           spark_python_task:
+             python_file: ./hello.py
+
+=== Authored scripts are removed without copying the policy script
+>>> [CLI] bundle validate -o json
+true
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.jobs.test_job
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]
+
+Destroy: 1 deleted
+
+>>> [CLI] cluster-policies delete [POLICY_ID]

acceptance/bundle/config-remote-sync/cluster_policy_replaces_array/policy.json.tmpl

@@ -0,0 +1,4 @@
+{
+  "name": "test-policy-$UNIQUE_NAME",
+  "definition": "{}"
+}

acceptance/bundle/config-remote-sync/cluster_policy_replaces_array/script

@@ -0,0 +1,34 @@
+export POLICY_ID=""
+bundle_configured=false
+cleanup() {
+    if $bundle_configured; then
+        errcode trace $CLI bundle destroy --auto-approve
+    fi
+    if [ -n "$POLICY_ID" ]; then
+        errcode trace $CLI cluster-policies delete "$POLICY_ID"
+    fi
+}
+trap cleanup EXIT
+
+POLICY_ID=$($CLI cluster-policies create --json "$(envsubst < policy.json.tmpl)" -o json | jq -er .policy_id)
+add_repl "$POLICY_ID" POLICY_ID
+envsubst < databricks.yml.tmpl > databricks.yml
+bundle_configured=true
+
+trace $CLI bundle deploy
+job_id=$(read_id.py test_job)
+
+title "Replace authored scripts with a script from an updated policy"
+trace $CLI cluster-policies edit --json "$(envsubst < updated-policy.json.tmpl)"
+edit_resource.py jobs "$job_id" <<'PY'
+r["tasks"][0]["new_cluster"]["init_scripts"] = [
+    {"volumes": {"destination": "/Volumes/catalog/schema/volume/policy.sh"}}
+]
+PY
+
+cp databricks.yml databricks.yml.backup
+trace $CLI bundle config-remote-sync --save
+trace diff.py databricks.yml.backup databricks.yml
+
+title "Authored scripts are removed without copying the policy script"
+trace $CLI bundle validate -o json | jq -e '.resources.jobs.test_job.tasks[0].new_cluster | has("init_scripts") | not'

acceptance/bundle/config-remote-sync/cluster_policy_replaces_array/test.toml

@@ -0,0 +1,2 @@
+RecordRequests = false
+Ignore = [".databricks", "databricks.yml", "databricks.yml.backup"]