Add Cluster Policy support to bundle resources

.nextchanges/bundles/cluster-policies.md

@@ -0,0 +1 @@
+Add support for the `cluster_policies` resource type in Declarative Automation Bundles. Cluster policies are only supported in direct deployment mode.

acceptance/bundle/invariant/configs/cluster_policy.yml.tmpl

@@ -0,0 +1,8 @@
+bundle:
+  name: test-bundle-$UNIQUE_NAME
+
+resources:
+  cluster_policies:
+    foo:
+      name: test-cluster-policy-$UNIQUE_NAME
+      definition: '{"spark_version":{"type":"fixed","value":"13.3.x-scala2.12"}}'

acceptance/bundle/invariant/continue_293/test.toml

@@ -16,6 +16,9 @@ EnvMatrixExclude.no_genie_space = ["INPUT_CONFIG=genie_space.yml.tmpl"]
 # instance_pools resource is not supported on v0.293.0
 EnvMatrixExclude.no_instance_pool = ["INPUT_CONFIG=instance_pool.yml.tmpl"]
 
+# cluster_policies resource is not supported on v0.293.0
+EnvMatrixExclude.no_cluster_policy = ["INPUT_CONFIG=cluster_policy.yml.tmpl"]
+
 # job_runs resource is not supported on v0.293.0
 EnvMatrixExclude.no_job_run = ["INPUT_CONFIG=job_run.yml.tmpl"]
 

acceptance/bundle/invariant/migrate/test.toml

@@ -14,6 +14,8 @@ EnvMatrixExclude.no_external_location = ["INPUT_CONFIG=external_location.yml.tmp
 EnvMatrixExclude.no_genie_space = ["INPUT_CONFIG=genie_space.yml.tmpl"]
 # Instance pools are direct-only; the terraform deploy that seeds the migration fails for them.
 EnvMatrixExclude.no_instance_pool = ["INPUT_CONFIG=instance_pool.yml.tmpl"]
+# Cluster policies are direct-only; the terraform deploy that seeds the migration fails for them.
+EnvMatrixExclude.no_cluster_policy = ["INPUT_CONFIG=cluster_policy.yml.tmpl"]
 
 # Cross-resource permission references (e.g. ${resources.jobs.job_b.permissions[0].level})
 # don't work in terraform mode: the terraform interpolator converts the path to

acceptance/bundle/invariant/test.toml

@@ -26,6 +26,7 @@ EnvMatrix.INPUT_CONFIG = [
     "catalog_optional_fields.yml.tmpl",
     "cluster.yml.tmpl",
     "cluster_apply_policy_default_values.yml.tmpl",
+    "cluster_policy.yml.tmpl",
     "dashboard.yml.tmpl",
     "job_apply_policy_default_values_job_cluster.yml.tmpl",
     "job_apply_policy_default_values_task_cluster.yml.tmpl",

acceptance/bundle/refschema/out.fields.txt

@@ -279,6 +279,40 @@ resources.catalogs.*.grants[*]	catalog.PrivilegeAssignment	ALL
 resources.catalogs.*.grants[*].principal	string	ALL
 resources.catalogs.*.grants[*].privileges	[]catalog.Privilege	ALL
 resources.catalogs.*.grants[*].privileges[*]	catalog.Privilege	ALL
+resources.cluster_policies.*.created_at_timestamp	int64	REMOTE
+resources.cluster_policies.*.creator_user_name	string	REMOTE
+resources.cluster_policies.*.definition	any	INPUT
+resources.cluster_policies.*.definition	string	ALL
+resources.cluster_policies.*.description	string	ALL
+resources.cluster_policies.*.id	string	INPUT
+resources.cluster_policies.*.is_default	bool	REMOTE
+resources.cluster_policies.*.libraries	[]compute.Library	ALL
+resources.cluster_policies.*.libraries[*]	compute.Library	ALL
+resources.cluster_policies.*.libraries[*].cran	*compute.RCranLibrary	ALL
+resources.cluster_policies.*.libraries[*].cran.package	string	ALL
+resources.cluster_policies.*.libraries[*].cran.repo	string	ALL
+resources.cluster_policies.*.libraries[*].egg	string	ALL
+resources.cluster_policies.*.libraries[*].jar	string	ALL
+resources.cluster_policies.*.libraries[*].maven	*compute.MavenLibrary	ALL
+resources.cluster_policies.*.libraries[*].maven.coordinates	string	ALL
+resources.cluster_policies.*.libraries[*].maven.exclusions	[]string	ALL
+resources.cluster_policies.*.libraries[*].maven.exclusions[*]	string	ALL
+resources.cluster_policies.*.libraries[*].maven.repo	string	ALL
+resources.cluster_policies.*.libraries[*].pypi	*compute.PythonPyPiLibrary	ALL
+resources.cluster_policies.*.libraries[*].pypi.package	string	ALL
+resources.cluster_policies.*.libraries[*].pypi.repo	string	ALL
+resources.cluster_policies.*.libraries[*].requirements	string	ALL
+resources.cluster_policies.*.libraries[*].whl	string	ALL
+resources.cluster_policies.*.lifecycle	resources.Lifecycle	INPUT
+resources.cluster_policies.*.lifecycle.prevent_destroy	bool	INPUT
+resources.cluster_policies.*.max_clusters_per_user	int64	ALL
+resources.cluster_policies.*.modified_status	string	INPUT
+resources.cluster_policies.*.name	string	ALL
+resources.cluster_policies.*.policy_family_definition_overrides	any	INPUT
+resources.cluster_policies.*.policy_family_definition_overrides	string	ALL
+resources.cluster_policies.*.policy_family_id	string	ALL
+resources.cluster_policies.*.policy_id	string	REMOTE
+resources.cluster_policies.*.url	string	INPUT
 resources.clusters.*.apply_policy_default_values	bool	ALL
 resources.clusters.*.autoscale	*compute.AutoScale	ALL
 resources.clusters.*.autoscale.max_workers	int	ALL

acceptance/bundle/resources/cluster_policies/basic/databricks.yml

@@ -0,0 +1,8 @@
+bundle:
+  name: test_cluster_policy
+
+resources:
+  cluster_policies:
+    test_cluster_policy:
+      name: my_cluster_policy
+      definition: '{"spark_version":{"type":"fixed","value":"13.3.x-scala2.12"}}'

acceptance/bundle/resources/cluster_policies/basic/output.txt

@@ -0,0 +1,125 @@
+
+>>> [CLI] bundle validate
+Name: test_cluster_policy
+Target: default
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/test_cluster_policy/default
+
+Validation OK!
+
+>>> [CLI] bundle validate -o json
+{
+  "test_cluster_policy": {
+    "definition": "{\"spark_version\":{\"type\":\"fixed\",\"value\":\"13.3.x-scala2.12\"}}",
+    "name": "my_cluster_policy"
+  }
+}
+
+>>> [CLI] bundle summary
+Name: test_cluster_policy
+Target: default
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/test_cluster_policy/default
+Resources:
+  Cluster Policies:
+    test_cluster_policy:
+      Name: my_cluster_policy
+      URL:  (not deployed)
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test_cluster_policy/default/files...
+Created cluster_policies.test_cluster_policy
+Files: 4 uploaded, 0 deleted
+Resources: 1 created, 0 changed, 0 deleted, 0 unchanged
+
+=== Verify the create request
+>>> jq select(.method == "POST" and (.path | contains("/policies/clusters/create"))) out.requests.txt
+{
+  "method": "POST",
+  "path": "/api/2.0/policies/clusters/create",
+  "body": {
+    "definition": "{\"spark_version\":{\"type\":\"fixed\",\"value\":\"13.3.x-scala2.12\"}}",
+    "name": "my_cluster_policy"
+  }
+}
+
+>>> [CLI] bundle summary
+Name: test_cluster_policy
+Target: default
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/test_cluster_policy/default
+Resources:
+  Cluster Policies:
+    test_cluster_policy:
+      Name: my_cluster_policy
+      URL:  [DATABRICKS_URL]/compute/policies/[UUID]?w=[NUMID]
+
+=== Update the cluster policy name
+>>> update_file.py databricks.yml my_cluster_policy my_cluster_policy_2
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test_cluster_policy/default/files...
+Updated cluster_policies.test_cluster_policy
+Files: 3 uploaded, 0 deleted
+Resources: 0 created, 1 changed, 0 deleted, 0 unchanged
+
+=== Verify the update request
+>>> jq select(.method == "POST" and (.path | contains("/policies/clusters/edit"))) out.requests.txt
+{
+  "method": "POST",
+  "path": "/api/2.0/policies/clusters/edit",
+  "body": {
+    "definition": "{\"spark_version\":{\"type\":\"fixed\",\"value\":\"13.3.x-scala2.12\"}}",
+    "name": "my_cluster_policy_2",
+    "policy_id": "[UUID]"
+  }
+}
+
+>>> [CLI] bundle summary
+Name: test_cluster_policy
+Target: default
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/test_cluster_policy/default
+Resources:
+  Cluster Policies:
+    test_cluster_policy:
+      Name: my_cluster_policy_2
+      URL:  [DATABRICKS_URL]/compute/policies/[UUID]?w=[NUMID]
+
+=== Destroy the cluster policy
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.cluster_policies.test_cluster_policy
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test_cluster_policy/default
+
+Destroy: 1 deleted
+
+=== Verify the destroy request
+>>> jq select(.method == "POST" and (.path | contains("/policies/clusters/delete"))) out.requests.txt
+{
+  "method": "POST",
+  "path": "/api/2.0/policies/clusters/delete",
+  "body": {
+    "policy_id": "[UUID]"
+  }
+}
+
+>>> [CLI] bundle summary
+Name: test_cluster_policy
+Target: default
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/test_cluster_policy/default
+Resources:
+  Cluster Policies:
+    test_cluster_policy:
+      Name: my_cluster_policy_2
+      URL:  (not deployed)
+
+>>> [CLI] bundle destroy --auto-approve
+No active deployment found to destroy!

acceptance/bundle/resources/cluster_policies/basic/script

@@ -0,0 +1,33 @@
+trace $CLI bundle validate
+trace $CLI bundle validate -o json | jq ".resources.cluster_policies"
+
+trace $CLI bundle summary
+
+cleanup() {
+    trace $CLI bundle destroy --auto-approve
+    rm out.requests.txt
+}
+trap cleanup EXIT
+trace $CLI bundle deploy
+
+title "Verify the create request"
+trace jq 'select(.method == "POST" and (.path | contains("/policies/clusters/create")))' out.requests.txt
+
+trace $CLI bundle summary
+
+title "Update the cluster policy name"
+trace update_file.py databricks.yml my_cluster_policy my_cluster_policy_2
+trace $CLI bundle deploy
+
+title "Verify the update request"
+trace jq 'select(.method == "POST" and (.path | contains("/policies/clusters/edit")))' out.requests.txt
+
+trace $CLI bundle summary
+
+title "Destroy the cluster policy"
+trace $CLI bundle destroy --auto-approve
+
+title "Verify the destroy request"
+trace jq 'select(.method == "POST" and (.path | contains("/policies/clusters/delete")))' out.requests.txt
+
+trace $CLI bundle summary

acceptance/bundle/resources/cluster_policies/definition_multiline/databricks.yml

@@ -0,0 +1,14 @@
+bundle:
+  name: cluster_policy_definition_multiline
+
+resources:
+  cluster_policies:
+    pol:
+      name: my_policy
+      definition: |-
+        {
+          "spark_version": {
+            "type": "fixed",
+            "value": "13.3.x-scala2.12"
+          }
+        }

acceptance/bundle/resources/cluster_policies/definition_multiline/output.txt

@@ -0,0 +1,33 @@
+
+>>> [CLI] bundle validate -o json
+{
+  "pol": {
+    "definition": "{\n  \"spark_version\": {\n    \"type\": \"fixed\",\n    \"value\": \"13.3.x-scala2.12\"\n  }\n}",
+    "name": "my_policy"
+  }
+}
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/cluster_policy_definition_multiline/default/files...
+Created cluster_policies.pol
+Files: 4 uploaded, 0 deleted
+Resources: 1 created, 0 changed, 0 deleted, 0 unchanged
+
+=== Create body preserves the block-scalar definition as a newline-escaped string
+>>> print_requests.py //policies/clusters
+{
+  "method": "POST",
+  "path": "/api/2.0/policies/clusters/create",
+  "body": {
+    "definition": "{\n  \"spark_version\": {\n    \"type\": \"fixed\",\n    \"value\": \"13.3.x-scala2.12\"\n  }\n}",
+    "name": "my_policy"
+  }
+}
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.cluster_policies.pol
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/cluster_policy_definition_multiline/default
+
+Destroy: 1 deleted

acceptance/bundle/resources/cluster_policies/definition_multiline/script

@@ -0,0 +1,9 @@
+trace $CLI bundle validate -o json | jq ".resources.cluster_policies"
+
+trace $CLI bundle deploy
+
+title "Create body preserves the block-scalar definition as a newline-escaped string"
+trace print_requests.py //policies/clusters
+
+trace $CLI bundle destroy --auto-approve
+rm out.requests.txt

acceptance/bundle/resources/cluster_policies/definition_yaml/databricks.yml

@@ -0,0 +1,18 @@
+bundle:
+  name: cluster_policy_definition_yaml
+
+resources:
+  cluster_policies:
+    pol:
+      name: my_policy
+      definition:
+        spark_version:
+          type: fixed
+          value: 13.3.x-scala2.12
+        autotermination_minutes:
+          type: range
+          minValue: 10
+          maxValue: 120
+        enable_elastic_disk:
+          type: fixed
+          value: true

acceptance/bundle/resources/cluster_policies/definition_yaml/output.txt

@@ -0,0 +1,33 @@
+
+>>> [CLI] bundle validate -o json
+{
+  "pol": {
+    "definition": "{\"autotermination_minutes\":{\"maxValue\":120,\"minValue\":10,\"type\":\"range\"},\"enable_elastic_disk\":{\"type\":\"fixed\",\"value\":true},\"spark_version\":{\"type\":\"fixed\",\"value\":\"13.3.x-scala2.12\"}}",
+    "name": "my_policy"
+  }
+}
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/cluster_policy_definition_yaml/default/files...
+Created cluster_policies.pol
+Files: 4 uploaded, 0 deleted
+Resources: 1 created, 0 changed, 0 deleted, 0 unchanged
+
+=== Native YAML definition serializes to compact JSON, preserving numbers and booleans as JSON types (not quoted strings)
+>>> print_requests.py //policies/clusters
+{
+  "method": "POST",
+  "path": "/api/2.0/policies/clusters/create",
+  "body": {
+    "definition": "{\"autotermination_minutes\":{\"maxValue\":120,\"minValue\":10,\"type\":\"range\"},\"enable_elastic_disk\":{\"type\":\"fixed\",\"value\":true},\"spark_version\":{\"type\":\"fixed\",\"value\":\"13.3.x-scala2.12\"}}",
+    "name": "my_policy"
+  }
+}
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.cluster_policies.pol
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/cluster_policy_definition_yaml/default
+
+Destroy: 1 deleted