diff --git a/.nextchanges/bundles/cluster-policies.md b/.nextchanges/bundles/cluster-policies.md new file mode 100644 index 00000000000..ace7f4b426b --- /dev/null +++ b/.nextchanges/bundles/cluster-policies.md @@ -0,0 +1 @@ +Add support for the `cluster_policies` resource type in Declarative Automation Bundles. Cluster policies are only supported in direct deployment mode. diff --git a/acceptance/bundle/invariant/configs/cluster_policy.yml.tmpl b/acceptance/bundle/invariant/configs/cluster_policy.yml.tmpl new file mode 100644 index 00000000000..aa514b0e0db --- /dev/null +++ b/acceptance/bundle/invariant/configs/cluster_policy.yml.tmpl @@ -0,0 +1,8 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + cluster_policies: + foo: + name: test-cluster-policy-$UNIQUE_NAME + definition: '{"spark_version":{"type":"fixed","value":"13.3.x-scala2.12"}}' diff --git a/acceptance/bundle/invariant/continue_293/test.toml b/acceptance/bundle/invariant/continue_293/test.toml index 1289c80af5e..f174ba144c4 100644 --- a/acceptance/bundle/invariant/continue_293/test.toml +++ b/acceptance/bundle/invariant/continue_293/test.toml @@ -16,6 +16,9 @@ EnvMatrixExclude.no_genie_space = ["INPUT_CONFIG=genie_space.yml.tmpl"] # instance_pools resource is not supported on v0.293.0 EnvMatrixExclude.no_instance_pool = ["INPUT_CONFIG=instance_pool.yml.tmpl"] +# cluster_policies resource is not supported on v0.293.0 +EnvMatrixExclude.no_cluster_policy = ["INPUT_CONFIG=cluster_policy.yml.tmpl"] + # job_runs resource is not supported on v0.293.0 EnvMatrixExclude.no_job_run = ["INPUT_CONFIG=job_run.yml.tmpl"] diff --git a/acceptance/bundle/invariant/delete_idempotent/out.test.toml b/acceptance/bundle/invariant/delete_idempotent/out.test.toml index 0ea874aac37..2dcac058e8c 100644 --- a/acceptance/bundle/invariant/delete_idempotent/out.test.toml +++ b/acceptance/bundle/invariant/delete_idempotent/out.test.toml @@ -8,6 +8,7 @@ EnvMatrix.INPUT_CONFIG = [ "catalog_optional_fields.yml.tmpl", "cluster.yml.tmpl", "cluster_apply_policy_default_values.yml.tmpl", + "cluster_policy.yml.tmpl", "dashboard.yml.tmpl", "job_apply_policy_default_values_job_cluster.yml.tmpl", "job_apply_policy_default_values_task_cluster.yml.tmpl", diff --git a/acceptance/bundle/invariant/destroy_idempotent/out.test.toml b/acceptance/bundle/invariant/destroy_idempotent/out.test.toml index 0ea874aac37..2dcac058e8c 100644 --- a/acceptance/bundle/invariant/destroy_idempotent/out.test.toml +++ b/acceptance/bundle/invariant/destroy_idempotent/out.test.toml @@ -8,6 +8,7 @@ EnvMatrix.INPUT_CONFIG = [ "catalog_optional_fields.yml.tmpl", "cluster.yml.tmpl", "cluster_apply_policy_default_values.yml.tmpl", + "cluster_policy.yml.tmpl", "dashboard.yml.tmpl", "job_apply_policy_default_values_job_cluster.yml.tmpl", "job_apply_policy_default_values_task_cluster.yml.tmpl", diff --git a/acceptance/bundle/invariant/migrate/test.toml b/acceptance/bundle/invariant/migrate/test.toml index cb927f4a69a..aa24bf58eed 100644 --- a/acceptance/bundle/invariant/migrate/test.toml +++ b/acceptance/bundle/invariant/migrate/test.toml @@ -14,6 +14,8 @@ EnvMatrixExclude.no_external_location = ["INPUT_CONFIG=external_location.yml.tmp EnvMatrixExclude.no_genie_space = ["INPUT_CONFIG=genie_space.yml.tmpl"] # Instance pools are direct-only; the terraform deploy that seeds the migration fails for them. EnvMatrixExclude.no_instance_pool = ["INPUT_CONFIG=instance_pool.yml.tmpl"] +# Cluster policies are direct-only; the terraform deploy that seeds the migration fails for them. +EnvMatrixExclude.no_cluster_policy = ["INPUT_CONFIG=cluster_policy.yml.tmpl"] # Cross-resource permission references (e.g. ${resources.jobs.job_b.permissions[0].level}) # don't work in terraform mode: the terraform interpolator converts the path to diff --git a/acceptance/bundle/invariant/no_drift/out.test.toml b/acceptance/bundle/invariant/no_drift/out.test.toml index 0ea874aac37..2dcac058e8c 100644 --- a/acceptance/bundle/invariant/no_drift/out.test.toml +++ b/acceptance/bundle/invariant/no_drift/out.test.toml @@ -8,6 +8,7 @@ EnvMatrix.INPUT_CONFIG = [ "catalog_optional_fields.yml.tmpl", "cluster.yml.tmpl", "cluster_apply_policy_default_values.yml.tmpl", + "cluster_policy.yml.tmpl", "dashboard.yml.tmpl", "job_apply_policy_default_values_job_cluster.yml.tmpl", "job_apply_policy_default_values_task_cluster.yml.tmpl", diff --git a/acceptance/bundle/invariant/test.toml b/acceptance/bundle/invariant/test.toml index faa2872a3b0..5c63aeef5b6 100644 --- a/acceptance/bundle/invariant/test.toml +++ b/acceptance/bundle/invariant/test.toml @@ -26,6 +26,7 @@ EnvMatrix.INPUT_CONFIG = [ "catalog_optional_fields.yml.tmpl", "cluster.yml.tmpl", "cluster_apply_policy_default_values.yml.tmpl", + "cluster_policy.yml.tmpl", "dashboard.yml.tmpl", "job_apply_policy_default_values_job_cluster.yml.tmpl", "job_apply_policy_default_values_task_cluster.yml.tmpl", diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index 6c2b033fa97..f04d1416677 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -279,6 +279,40 @@ resources.catalogs.*.grants[*] catalog.PrivilegeAssignment ALL resources.catalogs.*.grants[*].principal string ALL resources.catalogs.*.grants[*].privileges []catalog.Privilege ALL resources.catalogs.*.grants[*].privileges[*] catalog.Privilege ALL +resources.cluster_policies.*.created_at_timestamp int64 REMOTE +resources.cluster_policies.*.creator_user_name string REMOTE +resources.cluster_policies.*.definition any INPUT +resources.cluster_policies.*.definition string ALL +resources.cluster_policies.*.description string ALL +resources.cluster_policies.*.id string INPUT +resources.cluster_policies.*.is_default bool REMOTE +resources.cluster_policies.*.libraries []compute.Library ALL +resources.cluster_policies.*.libraries[*] compute.Library ALL +resources.cluster_policies.*.libraries[*].cran *compute.RCranLibrary ALL +resources.cluster_policies.*.libraries[*].cran.package string ALL +resources.cluster_policies.*.libraries[*].cran.repo string ALL +resources.cluster_policies.*.libraries[*].egg string ALL +resources.cluster_policies.*.libraries[*].jar string ALL +resources.cluster_policies.*.libraries[*].maven *compute.MavenLibrary ALL +resources.cluster_policies.*.libraries[*].maven.coordinates string ALL +resources.cluster_policies.*.libraries[*].maven.exclusions []string ALL +resources.cluster_policies.*.libraries[*].maven.exclusions[*] string ALL +resources.cluster_policies.*.libraries[*].maven.repo string ALL +resources.cluster_policies.*.libraries[*].pypi *compute.PythonPyPiLibrary ALL +resources.cluster_policies.*.libraries[*].pypi.package string ALL +resources.cluster_policies.*.libraries[*].pypi.repo string ALL +resources.cluster_policies.*.libraries[*].requirements string ALL +resources.cluster_policies.*.libraries[*].whl string ALL +resources.cluster_policies.*.lifecycle resources.Lifecycle INPUT +resources.cluster_policies.*.lifecycle.prevent_destroy bool INPUT +resources.cluster_policies.*.max_clusters_per_user int64 ALL +resources.cluster_policies.*.modified_status string INPUT +resources.cluster_policies.*.name string ALL +resources.cluster_policies.*.policy_family_definition_overrides any INPUT +resources.cluster_policies.*.policy_family_definition_overrides string ALL +resources.cluster_policies.*.policy_family_id string ALL +resources.cluster_policies.*.policy_id string REMOTE +resources.cluster_policies.*.url string INPUT resources.clusters.*.apply_policy_default_values bool ALL resources.clusters.*.autoscale *compute.AutoScale ALL resources.clusters.*.autoscale.max_workers int ALL diff --git a/acceptance/bundle/resources/cluster_policies/basic/databricks.yml b/acceptance/bundle/resources/cluster_policies/basic/databricks.yml new file mode 100644 index 00000000000..5156a2b9b0f --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/basic/databricks.yml @@ -0,0 +1,8 @@ +bundle: + name: test_cluster_policy + +resources: + cluster_policies: + test_cluster_policy: + name: my_cluster_policy + definition: '{"spark_version":{"type":"fixed","value":"13.3.x-scala2.12"}}' diff --git a/acceptance/bundle/resources/cluster_policies/basic/out.test.toml b/acceptance/bundle/resources/cluster_policies/basic/out.test.toml new file mode 100644 index 00000000000..0938e678987 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/basic/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/cluster_policies/basic/output.txt b/acceptance/bundle/resources/cluster_policies/basic/output.txt new file mode 100644 index 00000000000..735ccb05b6d --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/basic/output.txt @@ -0,0 +1,125 @@ + +>>> [CLI] bundle validate +Name: test_cluster_policy +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/test_cluster_policy/default + +Validation OK! + +>>> [CLI] bundle validate -o json +{ + "test_cluster_policy": { + "definition": "{\"spark_version\":{\"type\":\"fixed\",\"value\":\"13.3.x-scala2.12\"}}", + "name": "my_cluster_policy" + } +} + +>>> [CLI] bundle summary +Name: test_cluster_policy +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/test_cluster_policy/default +Resources: + Cluster Policies: + test_cluster_policy: + Name: my_cluster_policy + URL: (not deployed) + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test_cluster_policy/default/files... +Created cluster_policies.test_cluster_policy +Files: 4 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +=== Verify the create request +>>> jq select(.method == "POST" and (.path | contains("/policies/clusters/create"))) out.requests.txt +{ + "method": "POST", + "path": "/api/2.0/policies/clusters/create", + "body": { + "definition": "{\"spark_version\":{\"type\":\"fixed\",\"value\":\"13.3.x-scala2.12\"}}", + "name": "my_cluster_policy" + } +} + +>>> [CLI] bundle summary +Name: test_cluster_policy +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/test_cluster_policy/default +Resources: + Cluster Policies: + test_cluster_policy: + Name: my_cluster_policy + URL: [DATABRICKS_URL]/compute/policies/[UUID]?w=[NUMID] + +=== Update the cluster policy name +>>> update_file.py databricks.yml my_cluster_policy my_cluster_policy_2 + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test_cluster_policy/default/files... +Updated cluster_policies.test_cluster_policy +Files: 3 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged + +=== Verify the update request +>>> jq select(.method == "POST" and (.path | contains("/policies/clusters/edit"))) out.requests.txt +{ + "method": "POST", + "path": "/api/2.0/policies/clusters/edit", + "body": { + "definition": "{\"spark_version\":{\"type\":\"fixed\",\"value\":\"13.3.x-scala2.12\"}}", + "name": "my_cluster_policy_2", + "policy_id": "[UUID]" + } +} + +>>> [CLI] bundle summary +Name: test_cluster_policy +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/test_cluster_policy/default +Resources: + Cluster Policies: + test_cluster_policy: + Name: my_cluster_policy_2 + URL: [DATABRICKS_URL]/compute/policies/[UUID]?w=[NUMID] + +=== Destroy the cluster policy +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.cluster_policies.test_cluster_policy + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test_cluster_policy/default + +Destroy: 1 deleted + +=== Verify the destroy request +>>> jq select(.method == "POST" and (.path | contains("/policies/clusters/delete"))) out.requests.txt +{ + "method": "POST", + "path": "/api/2.0/policies/clusters/delete", + "body": { + "policy_id": "[UUID]" + } +} + +>>> [CLI] bundle summary +Name: test_cluster_policy +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/test_cluster_policy/default +Resources: + Cluster Policies: + test_cluster_policy: + Name: my_cluster_policy_2 + URL: (not deployed) + +>>> [CLI] bundle destroy --auto-approve +No active deployment found to destroy! diff --git a/acceptance/bundle/resources/cluster_policies/basic/script b/acceptance/bundle/resources/cluster_policies/basic/script new file mode 100644 index 00000000000..412bdcf1520 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/basic/script @@ -0,0 +1,33 @@ +trace $CLI bundle validate +trace $CLI bundle validate -o json | jq ".resources.cluster_policies" + +trace $CLI bundle summary + +cleanup() { + trace $CLI bundle destroy --auto-approve + rm out.requests.txt +} +trap cleanup EXIT +trace $CLI bundle deploy + +title "Verify the create request" +trace jq 'select(.method == "POST" and (.path | contains("/policies/clusters/create")))' out.requests.txt + +trace $CLI bundle summary + +title "Update the cluster policy name" +trace update_file.py databricks.yml my_cluster_policy my_cluster_policy_2 +trace $CLI bundle deploy + +title "Verify the update request" +trace jq 'select(.method == "POST" and (.path | contains("/policies/clusters/edit")))' out.requests.txt + +trace $CLI bundle summary + +title "Destroy the cluster policy" +trace $CLI bundle destroy --auto-approve + +title "Verify the destroy request" +trace jq 'select(.method == "POST" and (.path | contains("/policies/clusters/delete")))' out.requests.txt + +trace $CLI bundle summary diff --git a/acceptance/bundle/resources/cluster_policies/definition_multiline/databricks.yml b/acceptance/bundle/resources/cluster_policies/definition_multiline/databricks.yml new file mode 100644 index 00000000000..395e21aa5d0 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/definition_multiline/databricks.yml @@ -0,0 +1,14 @@ +bundle: + name: cluster_policy_definition_multiline + +resources: + cluster_policies: + pol: + name: my_policy + definition: |- + { + "spark_version": { + "type": "fixed", + "value": "13.3.x-scala2.12" + } + } diff --git a/acceptance/bundle/resources/cluster_policies/definition_multiline/out.test.toml b/acceptance/bundle/resources/cluster_policies/definition_multiline/out.test.toml new file mode 100644 index 00000000000..0938e678987 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/definition_multiline/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/cluster_policies/definition_multiline/output.txt b/acceptance/bundle/resources/cluster_policies/definition_multiline/output.txt new file mode 100644 index 00000000000..f43b38553a9 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/definition_multiline/output.txt @@ -0,0 +1,33 @@ + +>>> [CLI] bundle validate -o json +{ + "pol": { + "definition": "{\n \"spark_version\": {\n \"type\": \"fixed\",\n \"value\": \"13.3.x-scala2.12\"\n }\n}", + "name": "my_policy" + } +} + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/cluster_policy_definition_multiline/default/files... +Created cluster_policies.pol +Files: 4 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +=== Create body preserves the block-scalar definition as a newline-escaped string +>>> print_requests.py //policies/clusters +{ + "method": "POST", + "path": "/api/2.0/policies/clusters/create", + "body": { + "definition": "{\n \"spark_version\": {\n \"type\": \"fixed\",\n \"value\": \"13.3.x-scala2.12\"\n }\n}", + "name": "my_policy" + } +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.cluster_policies.pol + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/cluster_policy_definition_multiline/default + +Destroy: 1 deleted diff --git a/acceptance/bundle/resources/cluster_policies/definition_multiline/script b/acceptance/bundle/resources/cluster_policies/definition_multiline/script new file mode 100644 index 00000000000..c134016b6e7 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/definition_multiline/script @@ -0,0 +1,9 @@ +trace $CLI bundle validate -o json | jq ".resources.cluster_policies" + +trace $CLI bundle deploy + +title "Create body preserves the block-scalar definition as a newline-escaped string" +trace print_requests.py //policies/clusters + +trace $CLI bundle destroy --auto-approve +rm out.requests.txt diff --git a/acceptance/bundle/resources/cluster_policies/definition_yaml/databricks.yml b/acceptance/bundle/resources/cluster_policies/definition_yaml/databricks.yml new file mode 100644 index 00000000000..49ecc474a22 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/definition_yaml/databricks.yml @@ -0,0 +1,18 @@ +bundle: + name: cluster_policy_definition_yaml + +resources: + cluster_policies: + pol: + name: my_policy + definition: + spark_version: + type: fixed + value: 13.3.x-scala2.12 + autotermination_minutes: + type: range + minValue: 10 + maxValue: 120 + enable_elastic_disk: + type: fixed + value: true diff --git a/acceptance/bundle/resources/cluster_policies/definition_yaml/out.test.toml b/acceptance/bundle/resources/cluster_policies/definition_yaml/out.test.toml new file mode 100644 index 00000000000..0938e678987 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/definition_yaml/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/cluster_policies/definition_yaml/output.txt b/acceptance/bundle/resources/cluster_policies/definition_yaml/output.txt new file mode 100644 index 00000000000..aefc3e991fa --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/definition_yaml/output.txt @@ -0,0 +1,33 @@ + +>>> [CLI] bundle validate -o json +{ + "pol": { + "definition": "{\"autotermination_minutes\":{\"maxValue\":120,\"minValue\":10,\"type\":\"range\"},\"enable_elastic_disk\":{\"type\":\"fixed\",\"value\":true},\"spark_version\":{\"type\":\"fixed\",\"value\":\"13.3.x-scala2.12\"}}", + "name": "my_policy" + } +} + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/cluster_policy_definition_yaml/default/files... +Created cluster_policies.pol +Files: 4 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +=== Native YAML definition serializes to compact JSON, preserving numbers and booleans as JSON types (not quoted strings) +>>> print_requests.py //policies/clusters +{ + "method": "POST", + "path": "/api/2.0/policies/clusters/create", + "body": { + "definition": "{\"autotermination_minutes\":{\"maxValue\":120,\"minValue\":10,\"type\":\"range\"},\"enable_elastic_disk\":{\"type\":\"fixed\",\"value\":true},\"spark_version\":{\"type\":\"fixed\",\"value\":\"13.3.x-scala2.12\"}}", + "name": "my_policy" + } +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.cluster_policies.pol + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/cluster_policy_definition_yaml/default + +Destroy: 1 deleted diff --git a/acceptance/bundle/resources/cluster_policies/definition_yaml/script b/acceptance/bundle/resources/cluster_policies/definition_yaml/script new file mode 100644 index 00000000000..083c796fd5d --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/definition_yaml/script @@ -0,0 +1,9 @@ +trace $CLI bundle validate -o json | jq ".resources.cluster_policies" + +trace $CLI bundle deploy + +title "Native YAML definition serializes to compact JSON, preserving numbers and booleans as JSON types (not quoted strings)" +trace print_requests.py //policies/clusters + +trace $CLI bundle destroy --auto-approve +rm out.requests.txt diff --git a/acceptance/bundle/resources/cluster_policies/job_ref/databricks.yml b/acceptance/bundle/resources/cluster_policies/job_ref/databricks.yml new file mode 100644 index 00000000000..7bc49b780c2 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/job_ref/databricks.yml @@ -0,0 +1,24 @@ +bundle: + name: cluster_policy_job_ref + +resources: + cluster_policies: + pol: + name: my_policy + definition: |- + { + "spark_version": { + "type": "fixed", + "value": "13.3.x-scala2.12" + } + } + + jobs: + j: + name: my_job + tasks: + - task_key: main + new_cluster: + policy_id: ${resources.cluster_policies.pol.id} + spark_version: 13.3.x-scala2.12 + num_workers: 1 diff --git a/acceptance/bundle/resources/cluster_policies/job_ref/out.test.toml b/acceptance/bundle/resources/cluster_policies/job_ref/out.test.toml new file mode 100644 index 00000000000..0938e678987 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/job_ref/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/cluster_policies/job_ref/output.txt b/acceptance/bundle/resources/cluster_policies/job_ref/output.txt new file mode 100644 index 00000000000..2a80317d90e --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/job_ref/output.txt @@ -0,0 +1,124 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/cluster_policy_job_ref/default/files... +Created cluster_policies.pol +Created jobs.j +Files: 4 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +=== Deploy requests in dependency order: policy create precedes job create, job carries resolved policy id +>>> print_requests.py //policies/clusters //jobs +{ + "method": "POST", + "path": "/api/2.0/policies/clusters/create", + "body": { + "definition": "{\n \"spark_version\": {\n \"type\": \"fixed\",\n \"value\": \"13.3.x-scala2.12\"\n }\n}", + "name": "my_policy" + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/cluster_policy_job_ref/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "my_job", + "queue": { + "enabled": true + }, + "tasks": [ + { + "new_cluster": { + "num_workers": 1, + "policy_id": "[POL_ID]", + "spark_version": "13.3.x-scala2.12" + }, + "task_key": "main" + } + ] + } +} + +=== Update both the policy and the job +>>> update_file.py databricks.yml my_policy my_policy_2 + +>>> update_file.py databricks.yml my_job my_job_2 + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/cluster_policy_job_ref/default/files... +Updated cluster_policies.pol +Updated jobs.j +Files: 3 uploaded, 0 deleted +Resources: 0 created, 2 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //policies/clusters //jobs +{ + "method": "POST", + "path": "/api/2.0/policies/clusters/edit", + "body": { + "definition": "{\n \"spark_version\": {\n \"type\": \"fixed\",\n \"value\": \"13.3.x-scala2.12\"\n }\n}", + "name": "my_policy_2", + "policy_id": "[POL_ID]" + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/reset", + "body": { + "job_id": [NUMID], + "new_settings": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/cluster_policy_job_ref/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "my_job_2", + "queue": { + "enabled": true + }, + "tasks": [ + { + "new_cluster": { + "num_workers": 1, + "policy_id": "[POL_ID]", + "spark_version": "13.3.x-scala2.12" + }, + "task_key": "main" + } + ] + } + } +} + +=== Destroy: job delete precedes policy delete +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.cluster_policies.pol + delete resources.jobs.j + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/cluster_policy_job_ref/default + +Destroy: 2 deleted + +>>> print_requests.py //policies/clusters //jobs +{ + "method": "POST", + "path": "/api/2.2/jobs/delete", + "body": { + "job_id": [NUMID] + } +} +{ + "method": "POST", + "path": "/api/2.0/policies/clusters/delete", + "body": { + "policy_id": "[POL_ID]" + } +} diff --git a/acceptance/bundle/resources/cluster_policies/job_ref/script b/acceptance/bundle/resources/cluster_policies/job_ref/script new file mode 100644 index 00000000000..bd4b09777bb --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/job_ref/script @@ -0,0 +1,17 @@ +trace $CLI bundle deploy + +# Register [POL_ID] so the resolved reference in the job body is deterministic. +pol_id=`read_id.py pol` + +title "Deploy requests in dependency order: policy create precedes job create, job carries resolved policy id" +trace print_requests.py //policies/clusters //jobs + +title "Update both the policy and the job" +trace update_file.py databricks.yml my_policy my_policy_2 +trace update_file.py databricks.yml my_job my_job_2 +trace $CLI bundle deploy +trace print_requests.py //policies/clusters //jobs + +title "Destroy: job delete precedes policy delete" +trace $CLI bundle destroy --auto-approve +trace print_requests.py //policies/clusters //jobs diff --git a/acceptance/bundle/resources/cluster_policies/out_of_band_change/databricks.yml b/acceptance/bundle/resources/cluster_policies/out_of_band_change/databricks.yml new file mode 100644 index 00000000000..5156a2b9b0f --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/out_of_band_change/databricks.yml @@ -0,0 +1,8 @@ +bundle: + name: test_cluster_policy + +resources: + cluster_policies: + test_cluster_policy: + name: my_cluster_policy + definition: '{"spark_version":{"type":"fixed","value":"13.3.x-scala2.12"}}' diff --git a/acceptance/bundle/resources/cluster_policies/out_of_band_change/out.test.toml b/acceptance/bundle/resources/cluster_policies/out_of_band_change/out.test.toml new file mode 100644 index 00000000000..0938e678987 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/out_of_band_change/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/cluster_policies/out_of_band_change/output.txt b/acceptance/bundle/resources/cluster_policies/out_of_band_change/output.txt new file mode 100644 index 00000000000..897453ca753 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/out_of_band_change/output.txt @@ -0,0 +1,50 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test_cluster_policy/default/files... +Created cluster_policies.test_cluster_policy +Files: 5 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +=== Plan is a no-op immediately after deploy +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +=== Edit the policy definition out of band +>>> [CLI] cluster-policies edit --json {"policy_id":"[TEST_CLUSTER_POLICY_ID]","name":"my_cluster_policy","definition":"{\"spark_version\":{\"type\":\"fixed\",\"value\":\"14.3.x-scala2.12\"}}"} + +=== Plan detects the drift +>>> [CLI] bundle plan +update cluster_policies.test_cluster_policy + +Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged + +=== Redeploy reconciles the policy back to the configured definition +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test_cluster_policy/default/files... +Updated cluster_policies.test_cluster_policy +Files: 2 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged + +=== Verify the edit request restored the configured definition +>>> jq select(.method == "POST" and (.path | contains("/policies/clusters/edit"))) out.requests.txt +{ + "method": "POST", + "path": "/api/2.0/policies/clusters/edit", + "body": { + "definition": "{\"spark_version\":{\"type\":\"fixed\",\"value\":\"13.3.x-scala2.12\"}}", + "name": "my_cluster_policy", + "policy_id": "[TEST_CLUSTER_POLICY_ID]" + } +} + +=== Plan is a no-op again +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.cluster_policies.test_cluster_policy + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test_cluster_policy/default + +Destroy: 1 deleted diff --git a/acceptance/bundle/resources/cluster_policies/out_of_band_change/script b/acceptance/bundle/resources/cluster_policies/out_of_band_change/script new file mode 100644 index 00000000000..47d8585c72d --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/out_of_band_change/script @@ -0,0 +1,34 @@ +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +trace $CLI bundle deploy + +title "Plan is a no-op immediately after deploy" +trace $CLI bundle plan + +policy_id="$(read_id.py test_cluster_policy)" + +# Simulate an out-of-band change: edit the policy definition directly through the +# API, the way an admin would in the UI, without touching databricks.yml. The +# recorded bundle state is now stale, so the next plan must detect the drift. +title "Edit the policy definition out of band" +trace $CLI cluster-policies edit --json '{"policy_id":"'"$policy_id"'","name":"my_cluster_policy","definition":"{\"spark_version\":{\"type\":\"fixed\",\"value\":\"14.3.x-scala2.12\"}}"}' + +# Discard the out-of-band request so the verification below captures only the +# reconciling edit issued by the redeploy. +rm -f out.requests.txt + +title "Plan detects the drift" +trace $CLI bundle plan + +title "Redeploy reconciles the policy back to the configured definition" +trace $CLI bundle deploy + +title "Verify the edit request restored the configured definition" +trace jq 'select(.method == "POST" and (.path | contains("/policies/clusters/edit")))' out.requests.txt + +title "Plan is a no-op again" +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/cluster_policies/out_of_band_change/test.toml b/acceptance/bundle/resources/cluster_policies/out_of_band_change/test.toml new file mode 100644 index 00000000000..b03f24bc233 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/out_of_band_change/test.toml @@ -0,0 +1,4 @@ +Ignore = [ + ".databricks", + "databricks.yml", +] diff --git a/acceptance/bundle/resources/cluster_policies/policy_family_overrides/databricks.yml b/acceptance/bundle/resources/cluster_policies/policy_family_overrides/databricks.yml new file mode 100644 index 00000000000..a50210d97a6 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/policy_family_overrides/databricks.yml @@ -0,0 +1,34 @@ +bundle: + name: cluster_policy_overrides + +resources: + cluster_policies: + # Overrides as a compact JSON string. + json_string: + name: policy_json_string + policy_family_id: personal-vm + policy_family_definition_overrides: '{"autotermination_minutes":{"type":"fixed","value":30}}' + + # Overrides as a multiline block-scalar JSON string (preserved verbatim). + multiline: + name: policy_multiline + policy_family_id: personal-vm + policy_family_definition_overrides: |- + { + "autotermination_minutes": { + "type": "fixed", + "value": 30 + } + } + + # Overrides as native YAML (normalized to compact JSON, numbers/booleans kept as JSON types). + yaml: + name: policy_yaml + policy_family_id: personal-vm + policy_family_definition_overrides: + autotermination_minutes: + type: fixed + value: 30 + enable_elastic_disk: + type: fixed + value: true diff --git a/acceptance/bundle/resources/cluster_policies/policy_family_overrides/out.test.toml b/acceptance/bundle/resources/cluster_policies/policy_family_overrides/out.test.toml new file mode 100644 index 00000000000..0938e678987 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/policy_family_overrides/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/cluster_policies/policy_family_overrides/output.txt b/acceptance/bundle/resources/cluster_policies/policy_family_overrides/output.txt new file mode 100644 index 00000000000..3f87e0b314b --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/policy_family_overrides/output.txt @@ -0,0 +1,67 @@ + +>>> [CLI] bundle validate -o json +{ + "json_string": { + "name": "policy_json_string", + "policy_family_definition_overrides": "{\"autotermination_minutes\":{\"type\":\"fixed\",\"value\":30}}", + "policy_family_id": "personal-vm" + }, + "multiline": { + "name": "policy_multiline", + "policy_family_definition_overrides": "{\n \"autotermination_minutes\": {\n \"type\": \"fixed\",\n \"value\": 30\n }\n}", + "policy_family_id": "personal-vm" + }, + "yaml": { + "name": "policy_yaml", + "policy_family_definition_overrides": "{\"autotermination_minutes\":{\"type\":\"fixed\",\"value\":30},\"enable_elastic_disk\":{\"type\":\"fixed\",\"value\":true}}", + "policy_family_id": "personal-vm" + } +} + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/cluster_policy_overrides/default/files... +Created cluster_policies.json_string +Created cluster_policies.multiline +Created cluster_policies.yaml +Files: 4 uploaded, 0 deleted +Resources: 3 created, 0 changed, 0 deleted, 0 unchanged + +=== Create bodies carry policy_family_id and overrides as a JSON string (YAML normalized, JSON preserved) +>>> print_requests.py //policies/clusters/create --sort +{ + "method": "POST", + "path": "/api/2.0/policies/clusters/create", + "body": { + "name": "policy_json_string", + "policy_family_definition_overrides": "{\"autotermination_minutes\":{\"type\":\"fixed\",\"value\":30}}", + "policy_family_id": "personal-vm" + } +} +{ + "method": "POST", + "path": "/api/2.0/policies/clusters/create", + "body": { + "name": "policy_multiline", + "policy_family_definition_overrides": "{\n \"autotermination_minutes\": {\n \"type\": \"fixed\",\n \"value\": 30\n }\n}", + "policy_family_id": "personal-vm" + } +} +{ + "method": "POST", + "path": "/api/2.0/policies/clusters/create", + "body": { + "name": "policy_yaml", + "policy_family_definition_overrides": "{\"autotermination_minutes\":{\"type\":\"fixed\",\"value\":30},\"enable_elastic_disk\":{\"type\":\"fixed\",\"value\":true}}", + "policy_family_id": "personal-vm" + } +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.cluster_policies.json_string + delete resources.cluster_policies.multiline + delete resources.cluster_policies.yaml + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/cluster_policy_overrides/default + +Destroy: 3 deleted diff --git a/acceptance/bundle/resources/cluster_policies/policy_family_overrides/script b/acceptance/bundle/resources/cluster_policies/policy_family_overrides/script new file mode 100644 index 00000000000..fe1e5bd0507 --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/policy_family_overrides/script @@ -0,0 +1,11 @@ +trace $CLI bundle validate -o json | jq ".resources.cluster_policies" + +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT +trace $CLI bundle deploy + +title "Create bodies carry policy_family_id and overrides as a JSON string (YAML normalized, JSON preserved)" +trace print_requests.py //policies/clusters/create --sort diff --git a/acceptance/bundle/resources/cluster_policies/test.toml b/acceptance/bundle/resources/cluster_policies/test.toml new file mode 100644 index 00000000000..3fe510e7c1b --- /dev/null +++ b/acceptance/bundle/resources/cluster_policies/test.toml @@ -0,0 +1,6 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +Ignore = [ + "databricks.yml", +] diff --git a/acceptance/experimental/open/output.txt b/acceptance/experimental/open/output.txt index 1cd89cceda4..75591ed7423 100644 --- a/acceptance/experimental/open/output.txt +++ b/acceptance/experimental/open/output.txt @@ -9,13 +9,14 @@ === unknown resource type >>> [CLI] experimental open --url unknown 123 -Error: unknown resource type "unknown", must be one of: alerts, apps, catalogs, clusters, dashboards, database_catalogs, database_instances, experiments, genie_spaces, instance_pools, jobs, model_serving_endpoints, models, notebooks, pipelines, postgres_catalogs, postgres_synced_tables, quality_monitors, queries, registered_models, schemas, secrets, synced_database_tables, vector_search_endpoints, vector_search_indexes, volumes, warehouses +Error: unknown resource type "unknown", must be one of: alerts, apps, catalogs, cluster_policies, clusters, dashboards, database_catalogs, database_instances, experiments, genie_spaces, instance_pools, jobs, model_serving_endpoints, models, notebooks, pipelines, postgres_catalogs, postgres_synced_tables, quality_monitors, queries, registered_models, schemas, secrets, synced_database_tables, vector_search_endpoints, vector_search_indexes, volumes, warehouses === test auto-completion handler >>> [CLI] __complete experimental open , alerts apps catalogs +cluster_policies clusters dashboards database_catalogs diff --git a/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go b/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go index 46262ba8dbf..99ac6759ee8 100644 --- a/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go +++ b/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go @@ -36,6 +36,7 @@ var unsupportedResources = []string{ "vector_search_indexes", "job_runs", "secrets", + "cluster_policies", } func TestApplyBundlePermissions(t *testing.T) { diff --git a/bundle/config/mutator/resourcemutator/apply_presets.go b/bundle/config/mutator/resourcemutator/apply_presets.go index 72817d13566..f31866a6fc3 100644 --- a/bundle/config/mutator/resourcemutator/apply_presets.go +++ b/bundle/config/mutator/resourcemutator/apply_presets.go @@ -321,6 +321,16 @@ func (m *applyPresets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos } } + // Cluster Policies: Prefix. The policy name is a user-facing display name + // (unique, 1-100 chars), not the API id (policy_id), so prefixing it in dev + // mode avoids collisions between developers without changing identity. + for _, cp := range r.ClusterPolicies { + if cp == nil { + continue + } + cp.Name = prefix + cp.Name + } + // Vector Search Endpoints: no prefix. The endpoint name is the primary key // (it's what GET/UPDATE/DELETE address by), so prefixing it would change // the resource's identity rather than just its display name. diff --git a/bundle/config/mutator/resourcemutator/apply_target_mode_test.go b/bundle/config/mutator/resourcemutator/apply_target_mode_test.go index 53fdf89f50e..35eed05cc53 100644 --- a/bundle/config/mutator/resourcemutator/apply_target_mode_test.go +++ b/bundle/config/mutator/resourcemutator/apply_target_mode_test.go @@ -153,6 +153,9 @@ func mockBundle(mode config.Mode) *bundle.Bundle { InstancePools: map[string]*resources.InstancePool{ "instance_pool1": {CreateInstancePool: compute.CreateInstancePool{InstancePoolName: "instance_pool1", NodeTypeId: "i3.xlarge"}}, }, + ClusterPolicies: map[string]*resources.ClusterPolicy{ + "cluster_policy1": {CreatePolicy: compute.CreatePolicy{Name: "cluster_policy1"}}, + }, Dashboards: map[string]*resources.Dashboard{ "dashboard1": { DashboardConfig: resources.DashboardConfig{ diff --git a/bundle/config/mutator/resourcemutator/configure_cluster_policy_definition.go b/bundle/config/mutator/resourcemutator/configure_cluster_policy_definition.go new file mode 100644 index 00000000000..c0b1b4b5be7 --- /dev/null +++ b/bundle/config/mutator/resourcemutator/configure_cluster_policy_definition.go @@ -0,0 +1,71 @@ +package resourcemutator + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" +) + +// jsonPolicyFields are the JSON-policy fields normalized from inline YAML to a JSON string. +var jsonPolicyFields = []string{"definition", "policy_family_definition_overrides"} + +type configureClusterPolicyDefinition struct{} + +func ConfigureClusterPolicyDefinition() bundle.Mutator { + return &configureClusterPolicyDefinition{} +} + +func (c configureClusterPolicyDefinition) Name() string { + return "ConfigureClusterPolicyDefinition" +} + +func (c configureClusterPolicyDefinition) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { + var diags diag.Diagnostics + + pattern := dyn.NewPattern( + dyn.Key("resources"), + dyn.Key("cluster_policies"), + dyn.AnyKey(), + ) + + err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { + return dyn.MapByPattern(v, pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + for _, field := range jsonPolicyFields { + def := v.Get(field) + + // Marshal an inline structured value to a JSON string so both + // config-side and state-side carry the same plain string. Otherwise + // YAML decodes small ints as Go `int` while state JSON round-trip + // decodes them as `float64`, and structdiff reports false drift. + switch def.Kind() { + case dyn.KindInvalid, dyn.KindNil, dyn.KindString: + // KindInvalid means the field is absent; leave it for backend validation. + continue + case dyn.KindMap, dyn.KindSequence: + jsonBytes, err := json.Marshal(def.AsAny()) + if err != nil { + return dyn.InvalidValue, fmt.Errorf("failed to marshal inline %s: %w", field, err) + } + v, err = dyn.Set(v, field, dyn.V(string(jsonBytes))) + if err != nil { + return dyn.InvalidValue, err + } + default: + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("%s must be a string, map, or sequence, got %s", field, def.Kind()), + Locations: def.Locations(), + }) + } + } + return v, nil + }) + }) + + diags = diags.Extend(diag.FromErr(err)) + return diags +} diff --git a/bundle/config/mutator/resourcemutator/configure_cluster_policy_definition_test.go b/bundle/config/mutator/resourcemutator/configure_cluster_policy_definition_test.go new file mode 100644 index 00000000000..5e9d53fed4a --- /dev/null +++ b/bundle/config/mutator/resourcemutator/configure_cluster_policy_definition_test.go @@ -0,0 +1,111 @@ +package resourcemutator_test + +import ( + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/mutator/resourcemutator" + "github.com/databricks/cli/bundle/config/resources" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestConfigureClusterPolicyDefinition(t *testing.T) { + tests := []struct { + name string + field string // "definition" or "policy_family_definition_overrides" + value any + want any + // wantErr, when non-empty, is a substring expected in the diagnostics. + wantErr string + }{ + { + name: "definition: inline map is marshaled to a JSON string", + field: "definition", + value: map[string]any{"spark_version": map[string]any{"type": "fixed", "value": "13.3.x"}}, + want: `{"spark_version":{"type":"fixed","value":"13.3.x"}}`, + }, + { + name: "definition: inline sequence is marshaled to a JSON string", + field: "definition", + value: []any{"a", "b"}, + want: `["a","b"]`, + }, + { + name: "definition: inline string is left unchanged", + field: "definition", + value: `{"spark_version":{"type":"fixed"}}`, + want: `{"spark_version":{"type":"fixed"}}`, + }, + { + name: "definition: absent passes through", + field: "definition", + want: nil, + }, + { + name: "definition: non-structured is rejected", + field: "definition", + value: true, + wantErr: "definition must be a string, map, or sequence, got bool", + }, + { + // Number stays a JSON number (30, not "30"). + name: "overrides: inline map is marshaled to a JSON string", + field: "policy_family_definition_overrides", + value: map[string]any{"autotermination_minutes": map[string]any{"type": "fixed", "value": 30}}, + want: `{"autotermination_minutes":{"type":"fixed","value":30}}`, + }, + { + name: "overrides: inline string is left unchanged", + field: "policy_family_definition_overrides", + value: `{"autotermination_minutes":{"type":"fixed"}}`, + want: `{"autotermination_minutes":{"type":"fixed"}}`, + }, + { + name: "overrides: non-structured is rejected", + field: "policy_family_definition_overrides", + value: true, + wantErr: "policy_family_definition_overrides must be a string, map, or sequence, got bool", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cp := &resources.ClusterPolicy{} + switch tt.field { + case "policy_family_definition_overrides": + cp.PolicyFamilyDefinitionOverrides = tt.value + case "definition": + cp.Definition = tt.value + default: + t.Fatalf("unknown field %q", tt.field) + } + + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + ClusterPolicies: map[string]*resources.ClusterPolicy{"pol": cp}, + }, + }, + } + + diags := bundle.ApplySeq(t.Context(), b, resourcemutator.ConfigureClusterPolicyDefinition()) + + if tt.wantErr != "" { + require.Error(t, diags.Error()) + assert.ErrorContains(t, diags.Error(), tt.wantErr) + return + } + + require.NoError(t, diags.Error()) + got := b.Config.Resources.ClusterPolicies["pol"] + switch tt.field { + case "policy_family_definition_overrides": + assert.Equal(t, tt.want, got.PolicyFamilyDefinitionOverrides) + case "definition": + assert.Equal(t, tt.want, got.Definition) + } + }) + } +} diff --git a/bundle/config/mutator/resourcemutator/resource_mutator.go b/bundle/config/mutator/resourcemutator/resource_mutator.go index e8c33f0c59b..d771c0b3f57 100644 --- a/bundle/config/mutator/resourcemutator/resource_mutator.go +++ b/bundle/config/mutator/resourcemutator/resource_mutator.go @@ -208,6 +208,10 @@ func applyNormalizeMutators(ctx context.Context, b *bundle.Bundle) { // Updates (dynamic): resources.genie_spaces.*.serialized_space ConfigureGenieSpaceSerializedSpace(), + // Reads (dynamic): resources.cluster_policies.*.definition + // Updates (dynamic): resources.cluster_policies.*.definition (inline YAML -> JSON string) + ConfigureClusterPolicyDefinition(), + // Reads (typed): resources.alerts.*.file_path // Updates (typed): resources.alerts.* (loads alert configuration from .dbalert.json file) mutator.LoadDBAlertFiles(), diff --git a/bundle/config/mutator/resourcemutator/run_as_test.go b/bundle/config/mutator/resourcemutator/run_as_test.go index 87312608de8..be9abfe6836 100644 --- a/bundle/config/mutator/resourcemutator/run_as_test.go +++ b/bundle/config/mutator/resourcemutator/run_as_test.go @@ -31,41 +31,43 @@ func allResourceTypes(t *testing.T) []string { // Assert the total list of resource supported, as a sanity check that using // the dyn library gives us the correct list of all resources supported. Please // also update this check when adding a new resource - require.Equal(t, []string{ - "alerts", - "apps", - "catalogs", - "clusters", - "dashboards", - "database_catalogs", - "database_instances", - "experiments", - "external_locations", - "genie_spaces", - "instance_pools", - "job_runs", - "jobs", - "model_serving_endpoints", - "models", - "pipelines", - "postgres_branches", - "postgres_catalogs", - "postgres_databases", - "postgres_endpoints", - "postgres_projects", - "postgres_roles", - "postgres_synced_tables", - "quality_monitors", - "registered_models", - "schemas", - "secret_scopes", - "secrets", - "sql_warehouses", - "synced_database_tables", - "vector_search_endpoints", - "vector_search_indexes", - "volumes", - }, + require.Equal( + t, []string{ + "alerts", + "apps", + "catalogs", + "cluster_policies", + "clusters", + "dashboards", + "database_catalogs", + "database_instances", + "experiments", + "external_locations", + "genie_spaces", + "instance_pools", + "job_runs", + "jobs", + "model_serving_endpoints", + "models", + "pipelines", + "postgres_branches", + "postgres_catalogs", + "postgres_databases", + "postgres_endpoints", + "postgres_projects", + "postgres_roles", + "postgres_synced_tables", + "quality_monitors", + "registered_models", + "schemas", + "secret_scopes", + "secrets", + "sql_warehouses", + "synced_database_tables", + "vector_search_endpoints", + "vector_search_indexes", + "volumes", + }, resourceTypes, ) @@ -174,6 +176,7 @@ var allowList = []string{ "alerts", "catalogs", "clusters", + "cluster_policies", "dashboards", "database_catalogs", "database_instances", diff --git a/bundle/config/resources.go b/bundle/config/resources.go index ab12ec9f052..633332a78c0 100644 --- a/bundle/config/resources.go +++ b/bundle/config/resources.go @@ -45,6 +45,7 @@ type Resources struct { VectorSearchIndexes map[string]*resources.VectorSearchIndex `json:"vector_search_indexes,omitempty"` InstancePools map[string]*resources.InstancePool `json:"instance_pools,omitempty"` Secrets map[string]*resources.Secret `json:"secrets,omitempty"` + ClusterPolicies map[string]*resources.ClusterPolicy `json:"cluster_policies,omitempty"` } type ConfigResource interface { @@ -131,6 +132,7 @@ func (r *Resources) AllResources() []ResourceGroup { collectResourceMap(descriptions["vector_search_indexes"], r.VectorSearchIndexes), collectResourceMap(descriptions["instance_pools"], r.InstancePools), collectResourceMap(descriptions["secrets"], r.Secrets), + collectResourceMap(descriptions["cluster_policies"], r.ClusterPolicies), } } @@ -195,5 +197,6 @@ func SupportedResources() map[string]resources.ResourceDescription { "vector_search_endpoints": (&resources.VectorSearchEndpoint{}).ResourceDescription(), "vector_search_indexes": (&resources.VectorSearchIndex{}).ResourceDescription(), "secrets": (&resources.Secret{}).ResourceDescription(), + "cluster_policies": (&resources.ClusterPolicy{}).ResourceDescription(), } } diff --git a/bundle/config/resources/cluster_policy.go b/bundle/config/resources/cluster_policy.go new file mode 100644 index 00000000000..5c447b0beb9 --- /dev/null +++ b/bundle/config/resources/cluster_policy.go @@ -0,0 +1,67 @@ +package resources + +import ( + "context" + "net/url" + + "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/workspaceurls" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/marshal" + "github.com/databricks/databricks-sdk-go/service/compute" +) + +type ClusterPolicy struct { + BaseResource + compute.CreatePolicy + + // Shadows the embedded compute.CreatePolicy.Definition (a string). `any` lets the + // definition be authored as inline YAML; ConfigureClusterPolicyDefinition normalizes + // it to a JSON string before deploy. + Definition any `json:"definition,omitempty"` + + // Shadows the embedded compute.CreatePolicy.PolicyFamilyDefinitionOverrides (a string), + // same as Definition: also a policy document authorable as inline YAML. + PolicyFamilyDefinitionOverrides any `json:"policy_family_definition_overrides,omitempty"` +} + +func (s *ClusterPolicy) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, s) +} + +func (s ClusterPolicy) MarshalJSON() ([]byte, error) { + return marshal.Marshal(s) +} + +func (s *ClusterPolicy) Exists(ctx context.Context, w *databricks.WorkspaceClient, id string) (bool, error) { + _, err := w.ClusterPolicies.GetByPolicyId(ctx, id) + if err != nil { + log.Debugf(ctx, "cluster policy %s does not exist", id) + return false, err + } + return true, nil +} + +func (*ClusterPolicy) ResourceDescription() ResourceDescription { + return ResourceDescription{ + SingularName: "cluster_policy", + PluralName: "cluster_policies", + SingularTitle: "Cluster Policy", + PluralTitle: "Cluster Policies", + } +} + +func (s *ClusterPolicy) InitializeURL(baseURL url.URL) { + if s.ID == "" { + return + } + s.URL = workspaceurls.ResourceURL(baseURL, "cluster_policies", s.ID) +} + +func (s *ClusterPolicy) GetName() string { + return s.Name +} + +func (s *ClusterPolicy) GetURL() string { + return s.URL +} diff --git a/bundle/config/resources_test.go b/bundle/config/resources_test.go index d83f4da59b8..7e56f47a64a 100644 --- a/bundle/config/resources_test.go +++ b/bundle/config/resources_test.go @@ -204,6 +204,9 @@ func TestResourcesBindSupport(t *testing.T) { InstancePools: map[string]*resources.InstancePool{ "my_instance_pool": {}, }, + ClusterPolicies: map[string]*resources.ClusterPolicy{ + "my_cluster_policy": {}, + }, Dashboards: map[string]*resources.Dashboard{ "my_dashboard": {}, }, @@ -366,6 +369,7 @@ func TestResourcesBindSupport(t *testing.T) { m.GetMockSchemasAPI().EXPECT().GetByFullName(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockClustersAPI().EXPECT().GetByClusterId(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockInstancePoolsAPI().EXPECT().GetByInstancePoolId(mock.Anything, mock.Anything).Return(nil, nil) + m.GetMockClusterPoliciesAPI().EXPECT().GetByPolicyId(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockLakeviewAPI().EXPECT().Get(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockGenieAPI().EXPECT().GetSpace(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockVolumesAPI().EXPECT().Read(mock.Anything, mock.Anything).Return(nil, nil) diff --git a/bundle/deploy/terraform/lifecycle_test.go b/bundle/deploy/terraform/lifecycle_test.go index f8a5140576f..eaf3040f1c7 100644 --- a/bundle/deploy/terraform/lifecycle_test.go +++ b/bundle/deploy/terraform/lifecycle_test.go @@ -16,6 +16,7 @@ func TestConvertLifecycleForAllResources(t *testing.T) { // Resources that are only supported in direct mode and should not be converted to Terraform ignoredResources := []string{ "catalogs", + "cluster_policies", "external_locations", "genie_spaces", "instance_pools", diff --git a/bundle/direct/dresources/all.go b/bundle/direct/dresources/all.go index ad310468da0..2c82df2aab9 100644 --- a/bundle/direct/dresources/all.go +++ b/bundle/direct/dresources/all.go @@ -40,6 +40,7 @@ var SupportedResources = map[string]any{ "vector_search_indexes": (*ResourceVectorSearchIndex)(nil), "instance_pools": (*ResourceInstancePool)(nil), "secrets": (*ResourceSecret)(nil), + "cluster_policies": (*ResourceClusterPolicy)(nil), // Permissions "jobs.permissions": (*ResourcePermissions)(nil), diff --git a/bundle/direct/dresources/apitypes.generated.yml b/bundle/direct/dresources/apitypes.generated.yml index 5e61d803183..ec2e3c2519c 100644 --- a/bundle/direct/dresources/apitypes.generated.yml +++ b/bundle/direct/dresources/apitypes.generated.yml @@ -6,6 +6,8 @@ apps: apps.App catalogs: catalog.CreateCatalog +cluster_policies: compute.CreatePolicy + clusters: compute.ClusterSpec dashboards: dashboards.Dashboard diff --git a/bundle/direct/dresources/cluster_policy.go b/bundle/direct/dresources/cluster_policy.go new file mode 100644 index 00000000000..ae69390b9cc --- /dev/null +++ b/bundle/direct/dresources/cluster_policy.go @@ -0,0 +1,76 @@ +package dresources + +import ( + "context" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/utils" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/service/compute" +) + +type ResourceClusterPolicy struct { + client *databricks.WorkspaceClient +} + +func (*ResourceClusterPolicy) New(client *databricks.WorkspaceClient) *ResourceClusterPolicy { + return &ResourceClusterPolicy{client: client} +} + +func (*ResourceClusterPolicy) PrepareState(input *resources.ClusterPolicy) *compute.CreatePolicy { + cp := input.CreatePolicy + // Copy the shadow fields, already normalized to JSON strings by ConfigureClusterPolicyDefinition. + if s, ok := input.Definition.(string); ok { + cp.Definition = s + } + if s, ok := input.PolicyFamilyDefinitionOverrides.(string); ok { + cp.PolicyFamilyDefinitionOverrides = s + } + return &cp +} + +// RemapState copies the config fields shared by Policy and CreatePolicy; +// output-only fields (policy_id, created_at_timestamp, creator_user_name, is_default) are not in the state. +func (*ResourceClusterPolicy) RemapState(remote *compute.Policy) *compute.CreatePolicy { + return &compute.CreatePolicy{ + Definition: remote.Definition, + Description: remote.Description, + Libraries: remote.Libraries, + MaxClustersPerUser: remote.MaxClustersPerUser, + Name: remote.Name, + PolicyFamilyDefinitionOverrides: remote.PolicyFamilyDefinitionOverrides, + PolicyFamilyId: remote.PolicyFamilyId, + ForceSendFields: utils.FilterFields[compute.CreatePolicy](remote.ForceSendFields), + } +} + +func (r *ResourceClusterPolicy) DoRead(ctx context.Context, id string) (*compute.Policy, error) { + return r.client.ClusterPolicies.GetByPolicyId(ctx, id) +} + +func (r *ResourceClusterPolicy) DoCreate(ctx context.Context, config *compute.CreatePolicy) (string, *compute.Policy, error) { + resp, err := r.client.ClusterPolicies.Create(ctx, *config) + if err != nil { + return "", nil, err + } + + return resp.PolicyId, nil, nil +} + +func (r *ResourceClusterPolicy) DoUpdate(ctx context.Context, id string, config *compute.CreatePolicy, _ *PlanEntry) (*compute.Policy, error) { + return nil, r.client.ClusterPolicies.Edit(ctx, compute.EditPolicy{ + PolicyId: id, + Name: config.Name, + Definition: config.Definition, + Description: config.Description, + Libraries: config.Libraries, + MaxClustersPerUser: config.MaxClustersPerUser, + PolicyFamilyDefinitionOverrides: config.PolicyFamilyDefinitionOverrides, + PolicyFamilyId: config.PolicyFamilyId, + ForceSendFields: utils.FilterFields[compute.EditPolicy](config.ForceSendFields), + }) +} + +func (r *ResourceClusterPolicy) DoDelete(ctx context.Context, id string, _ *compute.CreatePolicy) error { + return r.client.ClusterPolicies.DeleteByPolicyId(ctx, id) +} diff --git a/bundle/direct/dresources/cluster_policy_test.go b/bundle/direct/dresources/cluster_policy_test.go new file mode 100644 index 00000000000..60377bfd0b3 --- /dev/null +++ b/bundle/direct/dresources/cluster_policy_test.go @@ -0,0 +1,47 @@ +package dresources + +import ( + "testing" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/stretchr/testify/assert" +) + +func TestClusterPolicyPrepareState(t *testing.T) { + tests := []struct { + name string + definition any + want string + }{ + { + // The normal post-mutator case: definition is already a JSON string. + name: "string definition is copied into state", + definition: `{"spark_version":{"type":"fixed"}}`, + want: `{"spark_version":{"type":"fixed"}}`, + }, + { + // ConfigureClusterPolicyDefinition guarantees a string, so a non-string + // is ignored rather than reaching the API. + name: "non-string definition is ignored", + definition: map[string]any{"spark_version": "fixed"}, + want: "", + }, + { + name: "absent definition leaves state empty", + definition: nil, + want: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + input := &resources.ClusterPolicy{Definition: tt.definition} + input.Name = "my_policy" + + got := (*ResourceClusterPolicy)(nil).PrepareState(input) + + assert.Equal(t, tt.want, got.Definition) + assert.Equal(t, "my_policy", got.Name) + }) + } +} diff --git a/bundle/direct/dresources/resources.generated.yml b/bundle/direct/dresources/resources.generated.yml index d5fb0d6c98d..bf63d30dc95 100644 --- a/bundle/direct/dresources/resources.generated.yml +++ b/bundle/direct/dresources/resources.generated.yml @@ -70,6 +70,8 @@ resources: # catalogs: no api field behaviors + # cluster_policies: no api field behaviors + # clusters: no api field behaviors dashboards: diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index 60bedb38d16..bd1c02857ad 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -543,6 +543,13 @@ resources: "azure_tenant_id": "description": |- PLACEHOLDER + "cluster_policies": + "description": |- + PLACEHOLDER + "$fields": + "lifecycle": + "description": |- + PLACEHOLDER "clusters": "description": |- The cluster definitions for the bundle, where each key is the name of a cluster. diff --git a/bundle/internal/validation/generated/required_fields.go b/bundle/internal/validation/generated/required_fields.go index 9268be0985f..97fa418097c 100644 --- a/bundle/internal/validation/generated/required_fields.go +++ b/bundle/internal/validation/generated/required_fields.go @@ -40,6 +40,10 @@ var RequiredFields = map[string][]string{ "resources.catalogs.*": {"name"}, "resources.catalogs.*.managed_encryption_settings.azure_encryption_settings": {"azure_tenant_id"}, + "resources.cluster_policies.*.libraries[*].cran": {"package"}, + "resources.cluster_policies.*.libraries[*].maven": {"coordinates"}, + "resources.cluster_policies.*.libraries[*].pypi": {"package"}, + "resources.clusters.*.cluster_log_conf.dbfs": {"destination"}, "resources.clusters.*.cluster_log_conf.s3": {"destination"}, "resources.clusters.*.cluster_log_conf.volumes": {"destination"}, diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index b4084979746..311426837f3 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -589,6 +589,51 @@ } ] }, + "resources.ClusterPolicy": { + "oneOf": [ + { + "type": "object", + "properties": { + "definition": { + "description": "Policy definition document expressed in [Databricks Cluster Policy Definition Language](https://docs.databricks.com/administration-guide/clusters/policy-definition.html).", + "$ref": "#/$defs/interface" + }, + "description": { + "description": "Additional human-readable description of the cluster policy.", + "$ref": "#/$defs/string" + }, + "libraries": { + "description": "A list of libraries to be installed on the next cluster restart that uses this policy. The maximum number of libraries is 500.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/compute.Library" + }, + "lifecycle": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Lifecycle" + }, + "max_clusters_per_user": { + "description": "Max number of clusters per user that can be active using this policy. If not present, there is no max limit.", + "$ref": "#/$defs/int64" + }, + "name": { + "description": "Cluster Policy name requested by the user. This has to be unique. Length must be between 1 and 100\ncharacters.", + "$ref": "#/$defs/string" + }, + "policy_family_definition_overrides": { + "description": "Policy definition JSON document expressed in [Databricks Policy Definition Language](https://docs.databricks.com/administration-guide/clusters/policy-definition.html).\nThe JSON document must be passed as a string and cannot be embedded in the requests.\n\nYou can use this to customize the policy definition inherited from the policy family.\nPolicy rules specified here are merged into the inherited policy definition.", + "$ref": "#/$defs/interface" + }, + "policy_family_id": { + "description": "ID of the policy family. The cluster policy's policy definition inherits the policy\nfamily's policy definition.\n\nCannot be used with `definition`. Use `policy_family_definition_overrides` instead to\ncustomize the policy definition.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "resources.Dashboard": { "oneOf": [ { @@ -3356,6 +3401,9 @@ "catalogs": { "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.Catalog" }, + "cluster_policies": { + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.ClusterPolicy" + }, "clusters": { "description": "The cluster definitions for the bundle, where each key is the name of a cluster.", "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.Cluster", @@ -14573,6 +14621,20 @@ } ] }, + "resources.ClusterPolicy": { + "oneOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.ClusterPolicy" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "resources.Dashboard": { "oneOf": [ { diff --git a/bundle/statemgmt/state_load_test.go b/bundle/statemgmt/state_load_test.go index b706b0770cf..ab39a4917b0 100644 --- a/bundle/statemgmt/state_load_test.go +++ b/bundle/statemgmt/state_load_test.go @@ -59,6 +59,7 @@ func TestStateToBundleEmptyLocalResources(t *testing.T) { "resources.vector_search_indexes.test_vector_search_index": {ID: "vs-index-1"}, "resources.instance_pools.test_instance_pool": {ID: "1"}, "resources.secrets.test_secret": {ID: "main.default.test_secret"}, + "resources.cluster_policies.test_cluster_policy": {ID: "cp-1"}, } err := StateToBundle(t.Context(), state, &config) assert.NoError(t, err) @@ -154,6 +155,9 @@ func TestStateToBundleEmptyLocalResources(t *testing.T) { assert.Equal(t, "1", config.Resources.InstancePools["test_instance_pool"].ID) assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.InstancePools["test_instance_pool"].ModifiedStatus) + assert.Equal(t, "cp-1", config.Resources.ClusterPolicies["test_cluster_policy"].ID) + assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.ClusterPolicies["test_cluster_policy"].ModifiedStatus) + assert.Equal(t, "main.default.test_secret", config.Resources.Secrets["test_secret"].ID) assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.Secrets["test_secret"].ModifiedStatus) @@ -402,6 +406,13 @@ func TestStateToBundleEmptyRemoteResources(t *testing.T) { }, }, }, + ClusterPolicies: map[string]*resources.ClusterPolicy{ + "test_cluster_policy": { + CreatePolicy: compute.CreatePolicy{ + Name: "test_cluster_policy", + }, + }, + }, }, } @@ -507,6 +518,9 @@ func TestStateToBundleEmptyRemoteResources(t *testing.T) { assert.Empty(t, config.Resources.InstancePools["test_instance_pool"].ID) assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.InstancePools["test_instance_pool"].ModifiedStatus) + assert.Empty(t, config.Resources.ClusterPolicies["test_cluster_policy"].ID) + assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.ClusterPolicies["test_cluster_policy"].ModifiedStatus) + AssertFullResourceCoverage(t, &config) } @@ -914,6 +928,18 @@ func TestStateToBundleModifiedResources(t *testing.T) { }, }, }, + ClusterPolicies: map[string]*resources.ClusterPolicy{ + "test_cluster_policy": { + CreatePolicy: compute.CreatePolicy{ + Name: "test_cluster_policy", + }, + }, + "test_cluster_policy_new": { + CreatePolicy: compute.CreatePolicy{ + Name: "test_cluster_policy_new", + }, + }, + }, }, } state := ExportedResourcesMap{ @@ -973,6 +999,8 @@ func TestStateToBundleModifiedResources(t *testing.T) { "resources.vector_search_indexes.test_vector_search_index_old": {ID: "vs-index-old"}, "resources.instance_pools.test_instance_pool": {ID: "1"}, "resources.instance_pools.test_instance_pool_old": {ID: "2"}, + "resources.cluster_policies.test_cluster_policy": {ID: "cp-1"}, + "resources.cluster_policies.test_cluster_policy_old": {ID: "cp-2"}, "resources.secrets.test_secret": {ID: "main.default.test_secret"}, "resources.secrets.test_secret_old": {ID: "main.default.test_secret_old"}, } @@ -1177,6 +1205,13 @@ func TestStateToBundleModifiedResources(t *testing.T) { assert.Empty(t, config.Resources.InstancePools["test_instance_pool_new"].ID) assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.InstancePools["test_instance_pool_new"].ModifiedStatus) + assert.Equal(t, "cp-1", config.Resources.ClusterPolicies["test_cluster_policy"].ID) + assert.Empty(t, config.Resources.ClusterPolicies["test_cluster_policy"].ModifiedStatus) + assert.Equal(t, "cp-2", config.Resources.ClusterPolicies["test_cluster_policy_old"].ID) + assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.ClusterPolicies["test_cluster_policy_old"].ModifiedStatus) + assert.Empty(t, config.Resources.ClusterPolicies["test_cluster_policy_new"].ID) + assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.ClusterPolicies["test_cluster_policy_new"].ModifiedStatus) + assert.Equal(t, "main.default.test_secret", config.Resources.Secrets["test_secret"].ID) assert.Empty(t, config.Resources.Secrets["test_secret"].ModifiedStatus) assert.Equal(t, "main.default.test_secret_old", config.Resources.Secrets["test_secret_old"].ID) diff --git a/cmd/experimental/workspace_open_test.go b/cmd/experimental/workspace_open_test.go index 71904677781..95502694472 100644 --- a/cmd/experimental/workspace_open_test.go +++ b/cmd/experimental/workspace_open_test.go @@ -67,7 +67,7 @@ func TestBuildWorkspaceURLFragmentBasedResources(t *testing.T) { func TestBuildWorkspaceURLUnknownResourceType(t *testing.T) { _, err := workspaceurls.BuildResourceURL("https://myworkspace.databricks.com", "unknown", "123", "") assert.ErrorContains(t, err, "unknown resource type \"unknown\"") - assert.ErrorContains(t, err, "alerts, apps, catalogs, clusters, dashboards, database_catalogs, database_instances, experiments, genie_spaces, instance_pools, jobs, model_serving_endpoints, models, notebooks, pipelines, postgres_catalogs, postgres_synced_tables, quality_monitors, queries, registered_models, schemas, secrets, synced_database_tables, vector_search_endpoints, vector_search_indexes, volumes, warehouses") + assert.ErrorContains(t, err, "alerts, apps, catalogs, cluster_policies, clusters, dashboards, database_catalogs, database_instances, experiments, genie_spaces, instance_pools, jobs, model_serving_endpoints, models, notebooks, pipelines, postgres_catalogs, postgres_synced_tables, quality_monitors, queries, registered_models, schemas, secrets, synced_database_tables, vector_search_endpoints, vector_search_indexes, volumes, warehouses") } func TestBuildWorkspaceURLHostWithTrailingSlash(t *testing.T) { @@ -110,6 +110,7 @@ func TestWorkspaceOpenCommandCompletion(t *testing.T) { "alerts", "apps", "catalogs", + "cluster_policies", "clusters", "dashboards", "database_catalogs", @@ -148,7 +149,7 @@ func TestWorkspaceOpenCommandCompletionSecondArg(t *testing.T) { func TestWorkspaceOpenCommandHelpText(t *testing.T) { cmd := newWorkspaceOpenCommand() - assert.Contains(t, cmd.Long, "Supported resource types: alerts, apps, catalogs, clusters, dashboards, database_catalogs, database_instances, experiments, genie_spaces, instance_pools, jobs, model_serving_endpoints, models, notebooks, pipelines, postgres_catalogs, postgres_synced_tables, quality_monitors, queries, registered_models, schemas, secrets, synced_database_tables, vector_search_endpoints, vector_search_indexes, volumes, warehouses.") + assert.Contains(t, cmd.Long, "Supported resource types: alerts, apps, catalogs, cluster_policies, clusters, dashboards, database_catalogs, database_instances, experiments, genie_spaces, instance_pools, jobs, model_serving_endpoints, models, notebooks, pipelines, postgres_catalogs, postgres_synced_tables, quality_monitors, queries, registered_models, schemas, secrets, synced_database_tables, vector_search_endpoints, vector_search_indexes, volumes, warehouses.") assert.Contains(t, cmd.Long, "databricks experimental open jobs 123456789") assert.Contains(t, cmd.Long, "databricks experimental open notebooks /Users/user@example.com/my-notebook") assert.Contains(t, cmd.Long, "databricks experimental open registered_models catalog.schema.my_model") diff --git a/libs/testserver/cluster_policies.go b/libs/testserver/cluster_policies.go new file mode 100644 index 00000000000..5616a32a00f --- /dev/null +++ b/libs/testserver/cluster_policies.go @@ -0,0 +1,98 @@ +package testserver + +import ( + "encoding/json" + "fmt" + "slices" + + "github.com/databricks/databricks-sdk-go/service/compute" +) + +func (s *FakeWorkspace) ClusterPoliciesCreate(req Request) any { + // Unmarshal into the stored (GET) type directly: CreatePolicy and Policy + // share JSON field names, so every config field is carried over. + var policy compute.Policy + if err := json.Unmarshal(req.Body, &policy); err != nil { + return Response{StatusCode: 400, Body: fmt.Sprintf("request parsing error: %s", err)} + } + + defer s.LockUnlock()() + + id := nextUUID() + policy.PolicyId = id + s.ClusterPolicies[id] = policy + + return Response{Body: compute.CreatePolicyResponse{PolicyId: id}} +} + +func (s *FakeWorkspace) ClusterPoliciesList(req Request) any { + defer s.LockUnlock()() + + ids := make([]string, 0, len(s.ClusterPolicies)) + for id := range s.ClusterPolicies { + ids = append(ids, id) + } + slices.Sort(ids) + + policies := make([]compute.Policy, 0, len(ids)) + for _, id := range ids { + policies = append(policies, s.ClusterPolicies[id]) + } + + return Response{Body: compute.ListPoliciesResponse{Policies: policies}} +} + +func (s *FakeWorkspace) ClusterPoliciesGet(req Request, policyId string) any { + defer s.LockUnlock()() + + policy, ok := s.ClusterPolicies[policyId] + if !ok { + return Response{StatusCode: 404} + } + + return Response{Body: policy} +} + +func (s *FakeWorkspace) ClusterPoliciesEdit(req Request) any { + var request compute.EditPolicy + if err := json.Unmarshal(req.Body, &request); err != nil { + return Response{StatusCode: 400, Body: fmt.Sprintf("request parsing error: %s", err)} + } + + defer s.LockUnlock()() + + policy, ok := s.ClusterPolicies[request.PolicyId] + if !ok { + return Response{StatusCode: 404} + } + + // Edit is a full replace of the writable fields; server-set fields + // (policy_id, created_at_timestamp, creator_user_name, is_default) are kept as stored. + policy.Name = request.Name + policy.Definition = request.Definition + policy.Description = request.Description + policy.Libraries = request.Libraries + policy.MaxClustersPerUser = request.MaxClustersPerUser + policy.PolicyFamilyDefinitionOverrides = request.PolicyFamilyDefinitionOverrides + policy.PolicyFamilyId = request.PolicyFamilyId + s.ClusterPolicies[request.PolicyId] = policy + + return Response{} +} + +func (s *FakeWorkspace) ClusterPoliciesDelete(req Request) any { + var request compute.DeletePolicy + if err := json.Unmarshal(req.Body, &request); err != nil { + return Response{StatusCode: 400, Body: fmt.Sprintf("request parsing error: %s", err)} + } + + defer s.LockUnlock()() + + if _, ok := s.ClusterPolicies[request.PolicyId]; !ok { + return Response{StatusCode: 404} + } + + delete(s.ClusterPolicies, request.PolicyId) + + return Response{} +} diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index 445417b3ff1..1982e13658a 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -196,6 +196,7 @@ type FakeWorkspace struct { ModelRegistryModelIDs map[string]string // model name -> numeric ID Clusters map[string]compute.ClusterDetails InstancePools map[string]compute.GetInstancePool + ClusterPolicies map[string]compute.Policy Catalogs map[string]catalog.CatalogInfo ExternalLocations map[string]catalog.ExternalLocationInfo RegisteredModels map[string]catalog.RegisteredModelInfo @@ -428,7 +429,13 @@ func NewFakeWorkspace(url, token string) *FakeWorkspace { SingleUserName: TestUser.UserName, }, }, - InstancePools: map[string]compute.GetInstancePool{}, + InstancePools: map[string]compute.GetInstancePool{}, + ClusterPolicies: map[string]compute.Policy{ + // Seeded so the stateful list keeps backing the variable-lookup tests + // (e.g. acceptance/bundle/variables/env_overrides resolves these by name). + "5678": {PolicyId: "5678", Name: "wrong-cluster-policy"}, + "9876": {PolicyId: "9876", Name: "some-test-cluster-policy"}, + }, VectorSearchIndexesPendingDeletion: map[string]int{}, } } diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index 082b771d1ea..05ac9df9370 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -27,18 +27,7 @@ var TestMetastore = catalog.MetastoreAssignment{ func AddDefaultHandlers(server *Server) { server.Handle("GET", "/api/2.0/policies/clusters/list", func(req Request) any { - return compute.ListPoliciesResponse{ - Policies: []compute.Policy{ - { - PolicyId: "5678", - Name: "wrong-cluster-policy", - }, - { - PolicyId: "9876", - Name: "some-test-cluster-policy", - }, - }, - } + return req.Workspace.ClusterPoliciesList(req) }) server.Handle("GET", "/api/2.0/instance-pools/list", func(req Request) any { @@ -62,6 +51,13 @@ func AddDefaultHandlers(server *Server) { return req.Workspace.InstancePoolsGet(req, req.URL.Query().Get("instance_pool_id")) }) + server.Handle("POST", "/api/2.0/policies/clusters/create", func(req Request) any { return req.Workspace.ClusterPoliciesCreate(req) }) + server.Handle("POST", "/api/2.0/policies/clusters/edit", func(req Request) any { return req.Workspace.ClusterPoliciesEdit(req) }) + server.Handle("POST", "/api/2.0/policies/clusters/delete", func(req Request) any { return req.Workspace.ClusterPoliciesDelete(req) }) + server.Handle("GET", "/api/2.0/policies/clusters/get", func(req Request) any { + return req.Workspace.ClusterPoliciesGet(req, req.URL.Query().Get("policy_id")) + }) + server.Handle("GET", "/api/2.1/clusters/list", func(req Request) any { return compute.ListClustersResponse{ Clusters: []compute.ClusterDetails{ diff --git a/libs/workspaceurls/urls.go b/libs/workspaceurls/urls.go index 4839c1e4273..61c3f271468 100644 --- a/libs/workspaceurls/urls.go +++ b/libs/workspaceurls/urls.go @@ -11,6 +11,7 @@ var resourceURLPatterns = map[string]string{ "alerts": "sql/alerts-v2/%s", "apps": "apps/%s", "catalogs": "explore/data/%s", + "cluster_policies": "compute/policies/%s", "clusters": "compute/clusters/%s", "dashboards": "dashboardsv3/%s/published", "database_catalogs": "explore/data/%s",