diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index 97fe7d96df..37bebc3f70 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -7,6 +7,7 @@ ### CLI ### Bundles +* engine/direct: Support bind & unbind. ([#4279](https://github.com/databricks/cli/pull/4279)) ### Dependency updates diff --git a/acceptance/bundle/deployment/bind/alert/out.test.toml b/acceptance/bundle/deployment/bind/alert/out.test.toml index e1a07763f6..ce10602d55 100644 --- a/acceptance/bundle/deployment/bind/alert/out.test.toml +++ b/acceptance/bundle/deployment/bind/alert/out.test.toml @@ -5,4 +5,4 @@ Cloud = true aws = false [EnvMatrix] - DATABRICKS_BUNDLE_ENGINE = ["terraform"] + DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/cluster/out.test.toml b/acceptance/bundle/deployment/bind/cluster/out.test.toml index b01b1f1f62..e28f520234 100644 --- a/acceptance/bundle/deployment/bind/cluster/out.test.toml +++ b/acceptance/bundle/deployment/bind/cluster/out.test.toml @@ -3,4 +3,4 @@ Cloud = true RequiresCluster = true [EnvMatrix] - DATABRICKS_BUNDLE_ENGINE = ["terraform"] + DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/dashboard/out.test.toml b/acceptance/bundle/deployment/bind/dashboard/out.test.toml index cae690414c..87248584bc 100644 --- a/acceptance/bundle/deployment/bind/dashboard/out.test.toml +++ b/acceptance/bundle/deployment/bind/dashboard/out.test.toml @@ -3,4 +3,4 @@ Cloud = true RequiresWarehouse = true [EnvMatrix] - DATABRICKS_BUNDLE_ENGINE = ["terraform"] + DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/dashboard/recreation/out.state_after_bind.direct.json b/acceptance/bundle/deployment/bind/dashboard/recreation/out.state_after_bind.direct.json new file mode 100644 index 0000000000..f839adb1cd --- /dev/null +++ b/acceptance/bundle/deployment/bind/dashboard/recreation/out.state_after_bind.direct.json @@ -0,0 +1,20 @@ +{ + "state_version": 1, + "cli_version": "[DEV_VERSION]", + "lineage": "[UUID]", + "serial": 2, + "state": { + "resources.dashboards.dashboard1": { + "__id__": "[DASHBOARD_ID]", + "state": { + "display_name": "test dashboard [UNIQUE_NAME]", + "embed_credentials": true, + "etag": [ETAG], + "parent_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/resources", + "published": true, + "serialized_dashboard": "{\"pages\":[{\"displayName\":\"Page One\",\"name\":\"02724bf2\"}]}", + "warehouse_id": "[TEST_DEFAULT_WAREHOUSE_ID]" + } + } + } +} diff --git a/acceptance/bundle/deployment/bind/dashboard/recreation/out.state_after_bind.terraform.json b/acceptance/bundle/deployment/bind/dashboard/recreation/out.state_after_bind.terraform.json new file mode 100644 index 0000000000..376db04bd0 --- /dev/null +++ b/acceptance/bundle/deployment/bind/dashboard/recreation/out.state_after_bind.terraform.json @@ -0,0 +1,42 @@ +{ + "version": 4, + "terraform_version": "1.5.5", + "serial": 1, + "lineage": "[UUID]", + "outputs": {}, + "resources": [ + { + "mode": "managed", + "type": "databricks_dashboard", + "name": "dashboard1", + "provider": "provider[\"registry.terraform.io/databricks/databricks\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "create_time": "[TIMESTAMP]", + "dashboard_change_detected": false, + "dashboard_id": "[DASHBOARD_ID]", + "dataset_catalog": null, + "dataset_schema": null, + "display_name": "test dashboard [UNIQUE_NAME]", + "embed_credentials": null, + "etag": [ETAG], + "file_path": null, + "id": "[DASHBOARD_ID]", + "lifecycle_state": "ACTIVE", + "md5": null, + "parent_path": "/Users/[USERNAME]", + "path": "/Users/[USERNAME]/test dashboard [UNIQUE_NAME].lvdash.json", + "serialized_dashboard": "{\"pages\":[{\"displayName\":\"Untitled page\",\"name\":\"02724bf2\",\"pageType\":\"PAGE_TYPE_CANVAS\"}]}", + "update_time": "[TIMESTAMP]", + "warehouse_id": "[TEST_DEFAULT_WAREHOUSE_ID]" + }, + "sensitive_attributes": [], + "private": "eyJzY2hlbWFfdmVyc2lvbiI6IjAifQ==" + } + ] + } + ], + "check_results": null +} diff --git a/acceptance/bundle/deployment/bind/dashboard/recreation/out.test.toml b/acceptance/bundle/deployment/bind/dashboard/recreation/out.test.toml index cae690414c..87248584bc 100644 --- a/acceptance/bundle/deployment/bind/dashboard/recreation/out.test.toml +++ b/acceptance/bundle/deployment/bind/dashboard/recreation/out.test.toml @@ -3,4 +3,4 @@ Cloud = true RequiresWarehouse = true [EnvMatrix] - DATABRICKS_BUNDLE_ENGINE = ["terraform"] + DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/dashboard/recreation/output.txt b/acceptance/bundle/deployment/bind/dashboard/recreation/output.txt index bab63ae4dd..99c26a8ccc 100644 --- a/acceptance/bundle/deployment/bind/dashboard/recreation/output.txt +++ b/acceptance/bundle/deployment/bind/dashboard/recreation/output.txt @@ -4,6 +4,14 @@ Updating deployment state... Successfully bound dashboard with an id '[DASHBOARD_ID]' Run 'bundle deploy' to deploy changes to your workspace +>>> print_state.py +[ETAG] + +>>> [CLI] bundle plan +recreate dashboards.dashboard1 + +Plan: 1 to add, 0 to change, 1 to delete, 0 unchanged + >>> errcode [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... diff --git a/acceptance/bundle/deployment/bind/dashboard/recreation/script b/acceptance/bundle/deployment/bind/dashboard/recreation/script index bff1482bd6..fbd7033f02 100644 --- a/acceptance/bundle/deployment/bind/dashboard/recreation/script +++ b/acceptance/bundle/deployment/bind/dashboard/recreation/script @@ -5,6 +5,7 @@ envsubst < databricks.yml.tmpl > databricks.yml # Create a pre-defined dashboard: DASHBOARD_ID=$($CLI lakeview create --display-name "${DASHBOARD_DISPLAY_NAME}" --warehouse-id "${TEST_DEFAULT_WAREHOUSE_ID}" --serialized-dashboard '{"pages":[{"name":"02724bf2","displayName":"Untitled page"}]}' | jq -r '.dashboard_id') +echo "$DASHBOARD_ID:DASHBOARD_ID" >> ACC_REPLS cleanupRemoveDashboard() { $CLI lakeview trash "${DASHBOARD_ID}" @@ -12,7 +13,13 @@ cleanupRemoveDashboard() { trap cleanupRemoveDashboard EXIT trace $CLI bundle deployment bind dashboard1 "${DASHBOARD_ID}" --auto-approve +trace print_state.py > out.state_after_bind.$DATABRICKS_BUNDLE_ENGINE.json +ETAG=$(jq '.state["resources.dashboards.dashboard1"].state.etag // .resources[0].instances[0].attributes.etag // "no-etag"' < out.state_after_bind.$DATABRICKS_BUNDLE_ENGINE.json) +echo $ETAG +add_repl.py $ETAG ETAG +json_in_json_normalize.py out.state_after_bind.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan trace errcode $CLI bundle deploy trace $CLI bundle deployment unbind dashboard1 diff --git a/acceptance/bundle/deployment/bind/database_instance/out.test.toml b/acceptance/bundle/deployment/bind/database_instance/out.test.toml index 90061dedb1..d560f1de04 100644 --- a/acceptance/bundle/deployment/bind/database_instance/out.test.toml +++ b/acceptance/bundle/deployment/bind/database_instance/out.test.toml @@ -2,4 +2,4 @@ Local = true Cloud = false [EnvMatrix] - DATABRICKS_BUNDLE_ENGINE = ["terraform"] + DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/experiment/out.get.direct.json b/acceptance/bundle/deployment/bind/experiment/out.get.direct.json new file mode 100644 index 0000000000..9ddcb0824e --- /dev/null +++ b/acceptance/bundle/deployment/bind/experiment/out.get.direct.json @@ -0,0 +1,4 @@ +{ + "name": "/Workspace/Users/[USERNAME]/test-experiment[UNIQUE_NAME]", + "lifecycle_stage": "active" +} diff --git a/acceptance/bundle/deployment/bind/experiment/out.get.terraform.json b/acceptance/bundle/deployment/bind/experiment/out.get.terraform.json new file mode 100644 index 0000000000..38934fd1ce --- /dev/null +++ b/acceptance/bundle/deployment/bind/experiment/out.get.terraform.json @@ -0,0 +1,4 @@ +{ + "name": "/Users/[USERNAME]/test-experiment[UNIQUE_NAME]", + "lifecycle_stage": "active" +} diff --git a/acceptance/bundle/deployment/bind/experiment/out.test.toml b/acceptance/bundle/deployment/bind/experiment/out.test.toml index a9f28de48a..d560f1de04 100644 --- a/acceptance/bundle/deployment/bind/experiment/out.test.toml +++ b/acceptance/bundle/deployment/bind/experiment/out.test.toml @@ -1,5 +1,5 @@ Local = true -Cloud = true +Cloud = false [EnvMatrix] - DATABRICKS_BUNDLE_ENGINE = ["terraform"] + DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/experiment/output.txt b/acceptance/bundle/deployment/bind/experiment/output.txt index 00bf54ef46..9636e52ae3 100644 --- a/acceptance/bundle/deployment/bind/experiment/output.txt +++ b/acceptance/bundle/deployment/bind/experiment/output.txt @@ -1,32 +1,27 @@ -=== Bind experiment test: -=== Substitute variables in the template -=== Create a pre-defined experiment -=== Bind experiment: Updating deployment state... +>>> [CLI] bundle deployment bind experiment1 [NUMID] --auto-approve +Updating deployment state... Successfully bound experiment with an id '[NUMID]' Run 'bundle deploy' to deploy changes to your workspace -=== Deploy bundle: Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +>>> [CLI] bundle deploy --force-lock --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Deploying resources... Updating deployment state... Deployment complete! -=== Read the pre-defined experiment: { - "name": "/Users/[USERNAME]/test-experiment[UNIQUE_NAME]", - "lifecycle_stage": "active" -} +>>> [CLI] experiments get-experiment [NUMID] -=== Unbind the experiment: Updating deployment state... +>>> [CLI] bundle deployment unbind experiment1 +Updating deployment state... -=== Destroy the bundle: All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default +>>> [CLI] bundle destroy --auto-approve +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default Deleting files... Destroy complete! -=== Read the pre-defined experiment again (expecting it still exists and is not deleted): { - "name": "/Users/[USERNAME]/test-experiment[UNIQUE_NAME]", - "lifecycle_stage": "active" -} +>>> [CLI] experiments get-experiment [NUMID] === Test cleanup: === Delete the pre-defined experiment: 0 diff --git a/acceptance/bundle/deployment/bind/experiment/script b/acceptance/bundle/deployment/bind/experiment/script index 24fadc3fff..be2bea428b 100644 --- a/acceptance/bundle/deployment/bind/experiment/script +++ b/acceptance/bundle/deployment/bind/experiment/script @@ -1,13 +1,8 @@ -title "Bind experiment test:" - -title "Substitute variables in the template" - # double slash at the start prevents Windows to apply replacements to the path EXPERIMENT_NAME="//Workspace/Users/${CURRENT_USER_NAME}/test-experiment$UNIQUE_NAME" export EXPERIMENT_NAME envsubst < databricks.yml.tmpl > databricks.yml -title "Create a pre-defined experiment" EXPERIMENT_ID=$($CLI experiments create-experiment "${EXPERIMENT_NAME}" | jq -r '.experiment_id') cleanupRemoveExperiment() { @@ -18,20 +13,16 @@ cleanupRemoveExperiment() { } trap cleanupRemoveExperiment EXIT -title "Bind experiment: " -$CLI bundle deployment bind experiment1 ${EXPERIMENT_ID} --auto-approve +trace $CLI bundle deployment bind experiment1 ${EXPERIMENT_ID} --auto-approve -title "Deploy bundle: " -$CLI bundle deploy --force-lock --auto-approve +trace $CLI bundle deploy --force-lock --auto-approve -title "Read the pre-defined experiment: " -$CLI experiments get-experiment ${EXPERIMENT_ID} | jq '{name: .experiment.name, lifecycle_stage: .experiment.lifecycle_stage}' +trace $CLI experiments get-experiment ${EXPERIMENT_ID} | jq '{name: .experiment.name, lifecycle_stage: .experiment.lifecycle_stage}' > out.get.$DATABRICKS_BUNDLE_ENGINE.json -title "Unbind the experiment: " -$CLI bundle deployment unbind experiment1 +trace $CLI bundle deployment unbind experiment1 -title "Destroy the bundle: " -$CLI bundle destroy --auto-approve +trace $CLI bundle destroy --auto-approve -title "Read the pre-defined experiment again (expecting it still exists and is not deleted): " -$CLI experiments get-experiment ${EXPERIMENT_ID} | jq '{name: .experiment.name, lifecycle_stage: .experiment.lifecycle_stage}' +trace $CLI experiments get-experiment ${EXPERIMENT_ID} | jq '{name: .experiment.name, lifecycle_stage: .experiment.lifecycle_stage}' > out.get2.$DATABRICKS_BUNDLE_ENGINE.json +diff out.get.$DATABRICKS_BUNDLE_ENGINE.json out.get2.$DATABRICKS_BUNDLE_ENGINE.json +rm out.get2.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/deployment/bind/experiment/test.toml b/acceptance/bundle/deployment/bind/experiment/test.toml index 0e8c8a3840..3ec944e783 100644 --- a/acceptance/bundle/deployment/bind/experiment/test.toml +++ b/acceptance/bundle/deployment/bind/experiment/test.toml @@ -1,2 +1,10 @@ +Badness = "Difference in GET request between direct and terraform; In direct, the prefix is /Workspace/Users, in TF it is /Users" Local = true -Cloud = true + +# Fails on Cloud with: +#=== CONT TestAccept/bundle/deployment/bind/experiment/DATABRICKS_BUNDLE_ENGINE=direct +# - "name": "/Workspace/Users/[USERNAME]/test-experiment[UNIQUE_NAME]", +# + "name": "/Users/[USERNAME]/test-experiment[UNIQUE_NAME]", +# https://github.com/databricks/cli/issues/4285 + +Cloud = false diff --git a/acceptance/bundle/deployment/bind/job/already-managed-different/out.bind.direct.txt b/acceptance/bundle/deployment/bind/job/already-managed-different/out.bind.direct.txt new file mode 100644 index 0000000000..6d32d71085 --- /dev/null +++ b/acceptance/bundle/deployment/bind/job/already-managed-different/out.bind.direct.txt @@ -0,0 +1,7 @@ + +>>> musterr [CLI] bundle deployment bind foo [NEW_JOB_ID] +Error: Resource already managed + +The bundle is already managing a resource for resources.jobs.foo with ID '[FOO_ID]'. +To bind to a different resource with ID '[NEW_JOB_ID]', you must first unbind the existing resource. + diff --git a/acceptance/bundle/deployment/bind/job/already-managed-different/out.test.toml b/acceptance/bundle/deployment/bind/job/already-managed-different/out.test.toml index 90061dedb1..d560f1de04 100644 --- a/acceptance/bundle/deployment/bind/job/already-managed-different/out.test.toml +++ b/acceptance/bundle/deployment/bind/job/already-managed-different/out.test.toml @@ -2,4 +2,4 @@ Local = true Cloud = false [EnvMatrix] - DATABRICKS_BUNDLE_ENGINE = ["terraform"] + DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/job/already-managed-different/script b/acceptance/bundle/deployment/bind/job/already-managed-different/script index 3508a57b8a..0d2b5ada11 100644 --- a/acceptance/bundle/deployment/bind/job/already-managed-different/script +++ b/acceptance/bundle/deployment/bind/job/already-managed-different/script @@ -1,5 +1,6 @@ # Bind job that is already bound to another ID trace $CLI bundle deploy +replace_ids.py new_job_id=$(trace $CLI jobs create --json '{"name": "My Job"}' | jq -r '.job_id') add_repl.py $new_job_id NEW_JOB_ID diff --git a/acceptance/bundle/deployment/bind/job/already-managed-different/test.toml b/acceptance/bundle/deployment/bind/job/already-managed-different/test.toml index 18b1a88417..2f62cd9beb 100644 --- a/acceptance/bundle/deployment/bind/job/already-managed-different/test.toml +++ b/acceptance/bundle/deployment/bind/job/already-managed-different/test.toml @@ -1 +1,4 @@ Cloud = false + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/job/already-managed-same/out.bind.direct.txt b/acceptance/bundle/deployment/bind/job/already-managed-same/out.bind.direct.txt new file mode 100644 index 0000000000..f36fe45fa5 --- /dev/null +++ b/acceptance/bundle/deployment/bind/job/already-managed-same/out.bind.direct.txt @@ -0,0 +1,6 @@ + +>>> musterr [CLI] bundle deployment bind foo [FOO_ID] +Error: Resource already managed + +The bundle is already managing resource for resources.jobs.foo and it is bound to the requested ID [FOO_ID]. + diff --git a/acceptance/bundle/deployment/bind/job/already-managed-same/out.test.toml b/acceptance/bundle/deployment/bind/job/already-managed-same/out.test.toml index 90061dedb1..d560f1de04 100644 --- a/acceptance/bundle/deployment/bind/job/already-managed-same/out.test.toml +++ b/acceptance/bundle/deployment/bind/job/already-managed-same/out.test.toml @@ -2,4 +2,4 @@ Local = true Cloud = false [EnvMatrix] - DATABRICKS_BUNDLE_ENGINE = ["terraform"] + DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/job/generate-and-bind/out.test.toml b/acceptance/bundle/deployment/bind/job/generate-and-bind/out.test.toml index 3cdb920b67..f474b1b917 100644 --- a/acceptance/bundle/deployment/bind/job/generate-and-bind/out.test.toml +++ b/acceptance/bundle/deployment/bind/job/generate-and-bind/out.test.toml @@ -2,4 +2,4 @@ Local = false Cloud = true [EnvMatrix] - DATABRICKS_BUNDLE_ENGINE = ["terraform"] + DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/job/job-abort-bind/out.test.toml b/acceptance/bundle/deployment/bind/job/job-abort-bind/out.test.toml index a9f28de48a..01ed6822af 100644 --- a/acceptance/bundle/deployment/bind/job/job-abort-bind/out.test.toml +++ b/acceptance/bundle/deployment/bind/job/job-abort-bind/out.test.toml @@ -2,4 +2,4 @@ Local = true Cloud = true [EnvMatrix] - DATABRICKS_BUNDLE_ENGINE = ["terraform"] + DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/job/job-abort-bind/output.txt b/acceptance/bundle/deployment/bind/job/job-abort-bind/output.txt index c05283b5ab..544448efbe 100644 --- a/acceptance/bundle/deployment/bind/job/job-abort-bind/output.txt +++ b/acceptance/bundle/deployment/bind/job/job-abort-bind/output.txt @@ -1,6 +1,6 @@ === Create a pre-defined job: -Created job with ID: [NUMID] +Created job with ID: [JOB_ID] === Expect binding to fail without an auto-approve flag: Error: This bind operation requires user confirmation, but the current console does not support prompting. Please specify --auto-approve if you would like to skip prompts and proceed. @@ -13,9 +13,9 @@ Updating deployment state... Deployment complete! === Check that job is not bound and not updated with config from bundle: ->>> [CLI] jobs get [NUMID] +>>> [CLI] jobs get [JOB_ID] { - "job_id": [NUMID], + "job_id": [JOB_ID], "settings": { "name": "test-unbound-job-[UNIQUE_NAME]", "tasks": [ @@ -29,4 +29,4 @@ Deployment complete! } } -=== Delete the pre-defined job [NUMID]:0 +=== Delete the pre-defined job [JOB_ID]:0 diff --git a/acceptance/bundle/deployment/bind/job/job-abort-bind/script b/acceptance/bundle/deployment/bind/job/job-abort-bind/script index 5ddb88e556..bbe22b68fc 100644 --- a/acceptance/bundle/deployment/bind/job/job-abort-bind/script +++ b/acceptance/bundle/deployment/bind/job/job-abort-bind/script @@ -23,6 +23,7 @@ JOB_ID=$($CLI jobs create --json ' }' | jq -r '.job_id') echo "Created job with ID: $JOB_ID" +echo "$JOB_ID:JOB_ID" >> ACC_REPLS envsubst < $TESTDIR/../job-spark-python-task/databricks.yml.tmpl > databricks.yml diff --git a/acceptance/bundle/deployment/bind/job/job-spark-python-task/out.test.toml b/acceptance/bundle/deployment/bind/job/job-spark-python-task/out.test.toml index a9f28de48a..01ed6822af 100644 --- a/acceptance/bundle/deployment/bind/job/job-spark-python-task/out.test.toml +++ b/acceptance/bundle/deployment/bind/job/job-spark-python-task/out.test.toml @@ -2,4 +2,4 @@ Local = true Cloud = true [EnvMatrix] - DATABRICKS_BUNDLE_ENGINE = ["terraform"] + DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/job/job-spark-python-task/output.txt b/acceptance/bundle/deployment/bind/job/job-spark-python-task/output.txt index 3ea190bbf8..3b68a53f08 100644 --- a/acceptance/bundle/deployment/bind/job/job-spark-python-task/output.txt +++ b/acceptance/bundle/deployment/bind/job/job-spark-python-task/output.txt @@ -28,8 +28,7 @@ Deployment complete! { "task_key": "my_notebook_task", "spark_python_task": { - "python_file": "/Workspace/Users/[USERNAME]/.bundle/test-bind-job-[UNIQUE_NAME]/files/hello_world.py", - "source": "WORKSPACE" + "python_file": "/Workspace/Users/[USERNAME]/.bundle/test-bind-job-[UNIQUE_NAME]/files/hello_world.py" } } ] @@ -60,8 +59,7 @@ Destroy complete! { "task_key": "my_notebook_task", "spark_python_task": { - "python_file": "/Workspace/Users/[USERNAME]/.bundle/test-bind-job-[UNIQUE_NAME]/files/hello_world.py", - "source": "WORKSPACE" + "python_file": "/Workspace/Users/[USERNAME]/.bundle/test-bind-job-[UNIQUE_NAME]/files/hello_world.py" } } ] diff --git a/acceptance/bundle/deployment/bind/job/job-spark-python-task/script b/acceptance/bundle/deployment/bind/job/job-spark-python-task/script index e18ceb002a..c083189941 100644 --- a/acceptance/bundle/deployment/bind/job/job-spark-python-task/script +++ b/acceptance/bundle/deployment/bind/job/job-spark-python-task/script @@ -41,7 +41,7 @@ title "Deploy bundle:" trace $CLI bundle deploy --force-lock --auto-approve title "Read the pre-defined job:" -trace $CLI jobs get $JOB_ID | jq '{job_id, settings: {name: .settings.name, tasks: [.settings.tasks[] | {task_key, spark_python_task: .spark_python_task}]}}' +trace $CLI jobs get $JOB_ID | jq '{job_id, settings: {name: .settings.name, tasks: [.settings.tasks[] | {task_key, spark_python_task: {python_file: .spark_python_task.python_file}}]}}' title "Unbind the job:" trace $CLI bundle deployment unbind foo @@ -53,4 +53,4 @@ title "Destroy the bundle:" trace $CLI bundle destroy --auto-approve title "Read the pre-defined job again (expecting it still exists):" -trace $CLI jobs get ${JOB_ID} | jq '{job_id, settings: {name: .settings.name, tasks: [.settings.tasks[] | {task_key, spark_python_task: .spark_python_task}]}}' +trace $CLI jobs get ${JOB_ID} | jq '{job_id, settings: {name: .settings.name, tasks: [.settings.tasks[] | {task_key, spark_python_task: {python_file: .spark_python_task.python_file}}]}}' diff --git a/acceptance/bundle/deployment/bind/job/noop-job/out.job.direct.json b/acceptance/bundle/deployment/bind/job/noop-job/out.job.direct.json new file mode 100644 index 0000000000..f566102545 --- /dev/null +++ b/acceptance/bundle/deployment/bind/job/noop-job/out.job.direct.json @@ -0,0 +1,22 @@ +{ + "created_time":[UNIX_TIME_MILLIS], + "creator_user_name":"[USERNAME]", + "job_id":[NUMID], + "run_as_user_name":"[USERNAME]", + "settings": { + "deployment": { + "kind":"BUNDLE", + "metadata_file_path":"/Workspace/Users/[USERNAME]/.bundle/my_project/default/state/metadata.json" + }, + "edit_mode":"UI_LOCKED", + "email_notifications": {}, + "format":"MULTI_TASK", + "max_concurrent_runs":1, + "name":"Updated Job", + "queue": { + "enabled":true + }, + "timeout_seconds":0, + "webhook_notifications": {} + } +} diff --git a/acceptance/bundle/deployment/bind/job/noop-job/out.job.terraform.json b/acceptance/bundle/deployment/bind/job/noop-job/out.job.terraform.json new file mode 100644 index 0000000000..53ac53e874 --- /dev/null +++ b/acceptance/bundle/deployment/bind/job/noop-job/out.job.terraform.json @@ -0,0 +1,25 @@ +{ + "created_time":[UNIX_TIME_MILLIS], + "creator_user_name":"[USERNAME]", + "job_id":[NUMID], + "run_as_user_name":"[USERNAME]", + "settings": { + "deployment": { + "kind":"BUNDLE", + "metadata_file_path":"/Workspace/Users/[USERNAME]/.bundle/my_project/default/state/metadata.json" + }, + "edit_mode":"UI_LOCKED", + "email_notifications": {}, + "format":"MULTI_TASK", + "max_concurrent_runs":1, + "name":"Updated Job", + "queue": { + "enabled":true + }, + "run_as": { + "user_name":"[USERNAME]" + }, + "timeout_seconds":0, + "webhook_notifications": {} + } +} diff --git a/acceptance/bundle/deployment/bind/job/noop-job/out.test.toml b/acceptance/bundle/deployment/bind/job/noop-job/out.test.toml index 90061dedb1..d560f1de04 100644 --- a/acceptance/bundle/deployment/bind/job/noop-job/out.test.toml +++ b/acceptance/bundle/deployment/bind/job/noop-job/out.test.toml @@ -2,4 +2,4 @@ Local = true Cloud = false [EnvMatrix] - DATABRICKS_BUNDLE_ENGINE = ["terraform"] + DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/job/noop-job/output.txt b/acceptance/bundle/deployment/bind/job/noop-job/output.txt index b44ce1f4fc..21d3780de6 100644 --- a/acceptance/bundle/deployment/bind/job/noop-job/output.txt +++ b/acceptance/bundle/deployment/bind/job/noop-job/output.txt @@ -13,28 +13,3 @@ Updating deployment state... Deployment complete! >>> [CLI] jobs get [NUMID] --output json -{ - "created_time":[UNIX_TIME_MILLIS], - "creator_user_name":"[USERNAME]", - "job_id":[NUMID], - "run_as_user_name":"[USERNAME]", - "settings": { - "deployment": { - "kind":"BUNDLE", - "metadata_file_path":"/Workspace/Users/[USERNAME]/.bundle/my_project/default/state/metadata.json" - }, - "edit_mode":"UI_LOCKED", - "email_notifications": {}, - "format":"MULTI_TASK", - "max_concurrent_runs":1, - "name":"Updated Job", - "queue": { - "enabled":true - }, - "run_as": { - "user_name":"[USERNAME]" - }, - "timeout_seconds":0, - "webhook_notifications": {} - } -} diff --git a/acceptance/bundle/deployment/bind/job/noop-job/script b/acceptance/bundle/deployment/bind/job/noop-job/script index 9afcae31b4..d844cea5b3 100644 --- a/acceptance/bundle/deployment/bind/job/noop-job/script +++ b/acceptance/bundle/deployment/bind/job/noop-job/script @@ -4,4 +4,5 @@ trace $CLI bundle deployment bind job_1 $job_id --auto-approve trace $CLI bundle deploy -trace $CLI jobs get $job_id --output json +# there is a difference: in terraform there is also run_as entry +trace $CLI jobs get $job_id --output json > out.job.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/deployment/bind/job/python-job/out.job.direct.json b/acceptance/bundle/deployment/bind/job/python-job/out.job.direct.json new file mode 100644 index 0000000000..f566102545 --- /dev/null +++ b/acceptance/bundle/deployment/bind/job/python-job/out.job.direct.json @@ -0,0 +1,22 @@ +{ + "created_time":[UNIX_TIME_MILLIS], + "creator_user_name":"[USERNAME]", + "job_id":[NUMID], + "run_as_user_name":"[USERNAME]", + "settings": { + "deployment": { + "kind":"BUNDLE", + "metadata_file_path":"/Workspace/Users/[USERNAME]/.bundle/my_project/default/state/metadata.json" + }, + "edit_mode":"UI_LOCKED", + "email_notifications": {}, + "format":"MULTI_TASK", + "max_concurrent_runs":1, + "name":"Updated Job", + "queue": { + "enabled":true + }, + "timeout_seconds":0, + "webhook_notifications": {} + } +} diff --git a/acceptance/bundle/deployment/bind/job/python-job/out.job.terraform.json b/acceptance/bundle/deployment/bind/job/python-job/out.job.terraform.json new file mode 100644 index 0000000000..53ac53e874 --- /dev/null +++ b/acceptance/bundle/deployment/bind/job/python-job/out.job.terraform.json @@ -0,0 +1,25 @@ +{ + "created_time":[UNIX_TIME_MILLIS], + "creator_user_name":"[USERNAME]", + "job_id":[NUMID], + "run_as_user_name":"[USERNAME]", + "settings": { + "deployment": { + "kind":"BUNDLE", + "metadata_file_path":"/Workspace/Users/[USERNAME]/.bundle/my_project/default/state/metadata.json" + }, + "edit_mode":"UI_LOCKED", + "email_notifications": {}, + "format":"MULTI_TASK", + "max_concurrent_runs":1, + "name":"Updated Job", + "queue": { + "enabled":true + }, + "run_as": { + "user_name":"[USERNAME]" + }, + "timeout_seconds":0, + "webhook_notifications": {} + } +} diff --git a/acceptance/bundle/deployment/bind/job/python-job/out.test.toml b/acceptance/bundle/deployment/bind/job/python-job/out.test.toml index 90061dedb1..d560f1de04 100644 --- a/acceptance/bundle/deployment/bind/job/python-job/out.test.toml +++ b/acceptance/bundle/deployment/bind/job/python-job/out.test.toml @@ -2,4 +2,4 @@ Local = true Cloud = false [EnvMatrix] - DATABRICKS_BUNDLE_ENGINE = ["terraform"] + DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/job/python-job/output.txt b/acceptance/bundle/deployment/bind/job/python-job/output.txt index 9ccd448692..2e327e3b47 100644 --- a/acceptance/bundle/deployment/bind/job/python-job/output.txt +++ b/acceptance/bundle/deployment/bind/job/python-job/output.txt @@ -13,28 +13,3 @@ Updating deployment state... Deployment complete! >>> [CLI] jobs get [NUMID] --output json -{ - "created_time":[UNIX_TIME_MILLIS], - "creator_user_name":"[USERNAME]", - "job_id":[NUMID], - "run_as_user_name":"[USERNAME]", - "settings": { - "deployment": { - "kind":"BUNDLE", - "metadata_file_path":"/Workspace/Users/[USERNAME]/.bundle/my_project/default/state/metadata.json" - }, - "edit_mode":"UI_LOCKED", - "email_notifications": {}, - "format":"MULTI_TASK", - "max_concurrent_runs":1, - "name":"Updated Job", - "queue": { - "enabled":true - }, - "run_as": { - "user_name":"[USERNAME]" - }, - "timeout_seconds":0, - "webhook_notifications": {} - } -} diff --git a/acceptance/bundle/deployment/bind/job/python-job/script b/acceptance/bundle/deployment/bind/job/python-job/script index 751921264b..8120a8a572 100644 --- a/acceptance/bundle/deployment/bind/job/python-job/script +++ b/acceptance/bundle/deployment/bind/job/python-job/script @@ -6,4 +6,4 @@ trace $UV_RUN $CLI bundle deployment bind job_1 $job_id --auto-approve trace $UV_RUN $CLI bundle deploy -trace $CLI jobs get $job_id --output json +trace $CLI jobs get $job_id --output json > out.job.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/deployment/bind/model-serving-endpoint/out.test.toml b/acceptance/bundle/deployment/bind/model-serving-endpoint/out.test.toml index a9f28de48a..01ed6822af 100644 --- a/acceptance/bundle/deployment/bind/model-serving-endpoint/out.test.toml +++ b/acceptance/bundle/deployment/bind/model-serving-endpoint/out.test.toml @@ -2,4 +2,4 @@ Local = true Cloud = true [EnvMatrix] - DATABRICKS_BUNDLE_ENGINE = ["terraform"] + DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/pipelines/recreate/databricks.yml b/acceptance/bundle/deployment/bind/pipelines/recreate/databricks.yml deleted file mode 100644 index 152942975f..0000000000 --- a/acceptance/bundle/deployment/bind/pipelines/recreate/databricks.yml +++ /dev/null @@ -1,11 +0,0 @@ -bundle: - name: test-pipeline-recreate - -resources: - pipelines: - foo: - name: test-pipeline - libraries: - - notebook: - path: ./nb.sql - catalog: main diff --git a/acceptance/bundle/deployment/bind/pipelines/recreate/databricks.yml.tmpl b/acceptance/bundle/deployment/bind/pipelines/recreate/databricks.yml.tmpl new file mode 100644 index 0000000000..25ce32ca56 --- /dev/null +++ b/acceptance/bundle/deployment/bind/pipelines/recreate/databricks.yml.tmpl @@ -0,0 +1,11 @@ +bundle: + name: test-pipeline-recreate-$UNIQUE_NAME + +resources: + pipelines: + foo: + name: test-pipeline-$UNIQUE_NAME + libraries: + - notebook: + path: ./nb.sql + storage: /Shared/new_storage diff --git a/acceptance/bundle/deployment/bind/pipelines/recreate/out.bind-fail.direct.txt b/acceptance/bundle/deployment/bind/pipelines/recreate/out.bind-fail.direct.txt new file mode 100644 index 0000000000..75d0788301 --- /dev/null +++ b/acceptance/bundle/deployment/bind/pipelines/recreate/out.bind-fail.direct.txt @@ -0,0 +1,14 @@ + +>>> musterr [CLI] bundle deployment bind foo [NEW_PIPELINE_ID] +Plan: recreate resources.pipelines.foo + +Changes detected: + ~ channel: null -> "CURRENT" + ~ deployment: null -> {"kind":"BUNDLE","metadata_file_path":"/Workspace/Users/[USERNAME]/.bundle/test-pipeline-recreate-[UNIQUE_NAME]/default/state/metadata.json"} + ~ edition: null -> "ADVANCED" + ~ libraries: [{"glob":{"include":"/Workspace/Users/someuser@databricks.com/lakeflow_pipeline/transformations/**"}},{"glob":{"include":"/Workspace/Users/foo@databricks.com/another/**"}}] -> [{"notebook":{"path":"/Workspace/Users/[USERNAME]/.bundle/test-pipeline-recreate-[UNIQUE_NAME]/default/files/nb"}}] + ~ name: "lakeflow-pipeline-[UNIQUE_NAME]" -> "test-pipeline-[UNIQUE_NAME]" + ~ storage: "/Shared/old_storage" -> "/Shared/new_storage" + +Error: This bind operation requires user confirmation, but the current console does not support prompting. Please specify --auto-approve if you would like to skip prompts and proceed. + diff --git a/acceptance/bundle/deployment/bind/pipelines/recreate/out.bind-fail.terraform.txt b/acceptance/bundle/deployment/bind/pipelines/recreate/out.bind-fail.terraform.txt index 667ffd2fd6..1d300160a9 100644 --- a/acceptance/bundle/deployment/bind/pipelines/recreate/out.bind-fail.terraform.txt +++ b/acceptance/bundle/deployment/bind/pipelines/recreate/out.bind-fail.terraform.txt @@ -11,7 +11,6 @@ Terraform will perform the following actions: # databricks_pipeline.foo must be replaced -/+ resource "databricks_pipeline" "foo" { - allow_duplicate_names = false -> null - ~ catalog = "old_catalog" -> "main" # forces replacement + cause = (known after apply) + channel = "CURRENT" + cluster_id = (known after apply) @@ -23,26 +22,26 @@ Terraform will perform the following actions: + health = (known after apply) ~ id = "[NEW_PIPELINE_ID]" -> (known after apply) ~ last_modified = [UNIX_TIME_MILLIS] -> (known after apply) - ~ name = "lakeflow-pipeline" -> "test-pipeline" + ~ name = "lakeflow-pipeline-[UNIQUE_NAME]" -> "test-pipeline-[UNIQUE_NAME]" - photon = false -> null - - root_path = "/Workspace/Users/[USERNAME]/lakeflow_pipeline" -> null + - root_path = "/Workspace/Users/someuser@databricks.com/lakeflow_pipeline" -> null ~ run_as_user_name = "[USERNAME]" -> (known after apply) - serverless = false -> null ~ state = "IDLE" -> (known after apply) - - storage = "old_storage" -> null # forces replacement + ~ storage = "/Shared/old_storage" -> "/Shared/new_storage" # forces replacement ~ url = "[DATABRICKS_URL]/#joblist/pipelines/[NEW_PIPELINE_ID]" -> (known after apply) + deployment { + kind = "BUNDLE" - + metadata_file_path = "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-recreate/default/state/metadata.json" + + metadata_file_path = "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-recreate-[UNIQUE_NAME]/default/state/metadata.json" } ~ library { - glob { - - include = "/Workspace/Users/[USERNAME]/lakeflow_pipeline/transformations/**" -> null + - include = "/Workspace/Users/someuser@databricks.com/lakeflow_pipeline/transformations/**" -> null } + notebook { - + path = "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-recreate/default/files/nb" + + path = "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-recreate-[UNIQUE_NAME]/default/files/nb" } } - library { diff --git a/acceptance/bundle/deployment/bind/pipelines/recreate/out.bind-success.direct.txt b/acceptance/bundle/deployment/bind/pipelines/recreate/out.bind-success.direct.txt new file mode 100644 index 0000000000..6c88a3134b --- /dev/null +++ b/acceptance/bundle/deployment/bind/pipelines/recreate/out.bind-success.direct.txt @@ -0,0 +1,5 @@ + +>>> [CLI] bundle deployment bind foo [NEW_PIPELINE_ID] --auto-approve +Updating deployment state... +Successfully bound pipeline with an id '[NEW_PIPELINE_ID]' +Run 'bundle deploy' to deploy changes to your workspace diff --git a/acceptance/bundle/deployment/bind/pipelines/recreate/out.deploy.requests.json b/acceptance/bundle/deployment/bind/pipelines/recreate/out.deploy.requests.json new file mode 100644 index 0000000000..b77f9d5138 --- /dev/null +++ b/acceptance/bundle/deployment/bind/pipelines/recreate/out.deploy.requests.json @@ -0,0 +1,25 @@ +{ + "method": "DELETE", + "path": "/api/2.0/pipelines/[NEW_PIPELINE_ID]" +} +{ + "method": "POST", + "path": "/api/2.0/pipelines", + "body": { + "channel": "CURRENT", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-recreate-[UNIQUE_NAME]/default/state/metadata.json" + }, + "edition": "ADVANCED", + "libraries": [ + { + "notebook": { + "path": "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-recreate-[UNIQUE_NAME]/default/files/nb" + } + } + ], + "name": "test-pipeline-[UNIQUE_NAME]", + "storage": "/Shared/new_storage" + } +} diff --git a/acceptance/bundle/deployment/bind/pipelines/recreate/out.summary.json b/acceptance/bundle/deployment/bind/pipelines/recreate/out.summary.json new file mode 100644 index 0000000000..59dbfe4a80 --- /dev/null +++ b/acceptance/bundle/deployment/bind/pipelines/recreate/out.summary.json @@ -0,0 +1,23 @@ +{ + "pipelines": { + "foo": { + "channel": "CURRENT", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-recreate-[UNIQUE_NAME]/default/state/metadata.json" + }, + "edition": "ADVANCED", + "id": "[NEW_PIPELINE_ID]", + "libraries": [ + { + "notebook": { + "path": "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-recreate-[UNIQUE_NAME]/default/files/nb" + } + } + ], + "name": "test-pipeline-[UNIQUE_NAME]", + "storage": "/Shared/new_storage", + "url": "[DATABRICKS_URL]/pipelines/[NEW_PIPELINE_ID]?o=[NUMID]" + } + } +} diff --git a/acceptance/bundle/deployment/bind/pipelines/recreate/out.test.toml b/acceptance/bundle/deployment/bind/pipelines/recreate/out.test.toml index 90061dedb1..01ed6822af 100644 --- a/acceptance/bundle/deployment/bind/pipelines/recreate/out.test.toml +++ b/acceptance/bundle/deployment/bind/pipelines/recreate/out.test.toml @@ -1,5 +1,5 @@ Local = true -Cloud = false +Cloud = true [EnvMatrix] - DATABRICKS_BUNDLE_ENGINE = ["terraform"] + DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/pipelines/recreate/output.txt b/acceptance/bundle/deployment/bind/pipelines/recreate/output.txt index b3d6303af1..af42322dee 100644 --- a/acceptance/bundle/deployment/bind/pipelines/recreate/output.txt +++ b/acceptance/bundle/deployment/bind/pipelines/recreate/output.txt @@ -1,4 +1,6 @@ +>>> print_requests.py ^//import-file/ ^//workspace/ + >>> [CLI] bundle summary -o json >>> [CLI] bundle plan @@ -7,7 +9,7 @@ recreate pipelines.foo Plan: 1 to add, 0 to change, 1 to delete, 0 unchanged >>> musterr [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-recreate/default/files... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-recreate-[UNIQUE_NAME]/default/files... This action will result in the deletion or recreation of the following Lakeflow Spark Declarative Pipelines along with the Streaming Tables (STs) and Materialized Views (MVs) managed by them. Recreating the pipelines will @@ -18,7 +20,7 @@ Error: the deployment requires destructive actions, but current console does not >>> [CLI] bundle deploy --auto-approve -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-recreate/default/files... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-recreate-[UNIQUE_NAME]/default/files... This action will result in the deletion or recreation of the following Lakeflow Spark Declarative Pipelines along with the Streaming Tables (STs) and Materialized Views (MVs) managed by them. Recreating the pipelines will @@ -29,4 +31,4 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py ^//import-file/ ^//workspace/delete ^//telemetry-ext +>>> print_requests.py ^//import-file/ ^//workspace/ ^//telemetry-ext diff --git a/acceptance/bundle/deployment/bind/pipelines/recreate/pipeline.json b/acceptance/bundle/deployment/bind/pipelines/recreate/pipeline.json deleted file mode 100644 index d022085ea9..0000000000 --- a/acceptance/bundle/deployment/bind/pipelines/recreate/pipeline.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "name": "lakeflow-pipeline", - "catalog": "old_catalog", - "storage": "old_storage", - "libraries": [ - { - "glob": { - "include": "/Workspace/Users/tester@databricks.com/lakeflow_pipeline/transformations/**" - } - }, - { - "glob": { - "include": "/Workspace/Users/foo@databricks.com/another/**" - } - } - ], - "root_path": "/Workspace/Users/tester@databricks.com/lakeflow_pipeline" -} diff --git a/acceptance/bundle/deployment/bind/pipelines/recreate/pipeline.json.tmpl b/acceptance/bundle/deployment/bind/pipelines/recreate/pipeline.json.tmpl new file mode 100644 index 0000000000..f3be2164ab --- /dev/null +++ b/acceptance/bundle/deployment/bind/pipelines/recreate/pipeline.json.tmpl @@ -0,0 +1,17 @@ +{ + "name": "lakeflow-pipeline-$UNIQUE_NAME", + "storage": "/Shared/old_storage", + "libraries": [ + { + "glob": { + "include": "/Workspace/Users/someuser@databricks.com/lakeflow_pipeline/transformations/**" + } + }, + { + "glob": { + "include": "/Workspace/Users/foo@databricks.com/another/**" + } + } + ], + "root_path": "/Workspace/Users/someuser@databricks.com/lakeflow_pipeline" +} diff --git a/acceptance/bundle/deployment/bind/pipelines/recreate/script b/acceptance/bundle/deployment/bind/pipelines/recreate/script index 542c3182d0..aba4cfb97a 100644 --- a/acceptance/bundle/deployment/bind/pipelines/recreate/script +++ b/acceptance/bundle/deployment/bind/pipelines/recreate/script @@ -1,19 +1,22 @@ +envsubst < databricks.yml.tmpl > databricks.yml +envsubst < pipeline.json.tmpl > pipeline.json + NEW_PIPELINE_ID=$($CLI pipelines create --json @pipeline.json | jq -r .pipeline_id) add_repl.py $NEW_PIPELINE_ID NEW_PIPELINE_ID rm -f out.requests.txt trace musterr $CLI bundle deployment bind foo $NEW_PIPELINE_ID &> out.bind-fail.$DATABRICKS_BUNDLE_ENGINE.txt -print_requests.py '^//import-file/' '^//workspace/delete' +print_requests.py '^//import-file/' '^//workspace/' rm -f out.requests.txt trace $CLI bundle deployment bind foo $NEW_PIPELINE_ID --auto-approve &> out.bind-success.$DATABRICKS_BUNDLE_ENGINE.txt -print_requests.py '^//import-file/' '^//workspace/delete' +trace print_requests.py '^//import-file/' '^//workspace/' -trace $CLI bundle summary -o json > out.summary.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle summary -o json | jq .resources > out.summary.json trace $CLI bundle plan trace musterr $CLI bundle deploy rm -f out.requests.txt trace $CLI bundle deploy --auto-approve -trace print_requests.py '^//import-file/' '^//workspace/delete' '^//telemetry-ext' > out.deploy.requests.$DATABRICKS_BUNDLE_ENGINE.json +trace print_requests.py '^//import-file/' '^//workspace/' '^//telemetry-ext' > out.deploy.requests.json diff --git a/acceptance/bundle/deployment/bind/pipelines/recreate/test.toml b/acceptance/bundle/deployment/bind/pipelines/recreate/test.toml index eb67288687..ded8808b22 100644 --- a/acceptance/bundle/deployment/bind/pipelines/recreate/test.toml +++ b/acceptance/bundle/deployment/bind/pipelines/recreate/test.toml @@ -1,2 +1,3 @@ +Cloud = true RecordRequests = true -Ignore = [".databricks"] +Ignore = [".databricks", "pipeline.json"] diff --git a/acceptance/bundle/deployment/bind/pipelines/update/out.bind-fail.direct.txt b/acceptance/bundle/deployment/bind/pipelines/update/out.bind-fail.direct.txt new file mode 100644 index 0000000000..385516aec5 --- /dev/null +++ b/acceptance/bundle/deployment/bind/pipelines/update/out.bind-fail.direct.txt @@ -0,0 +1,14 @@ + +>>> musterr [CLI] bundle deployment bind foo [NEW_PIPELINE_ID] +Plan: update resources.pipelines.foo + +Changes detected: + ~ catalog: null -> "main" + ~ channel: null -> "CURRENT" + ~ deployment: null -> {"kind":"BUNDLE","metadata_file_path":"/Workspace/Users/[USERNAME]/.bundle/test-pipeline-recreate/default/state/metadata.json"} + ~ edition: null -> "ADVANCED" + ~ libraries: [{"glob":{"include":"/Workspace/Users/[USERNAME]/lakeflow_pipeline/transformations/**"}},{"glob":{"include":"/Workspace/Users/foo@databricks.com/another/**"}}] -> [{"notebook":{"path":"/Workspace/Users/[USERNAME]/.bundle/test-pipeline-recreate/default/files/nb"}}] + ~ name: "lakeflow-pipeline" -> "test-pipeline" + +Error: This bind operation requires user confirmation, but the current console does not support prompting. Please specify --auto-approve if you would like to skip prompts and proceed. + diff --git a/acceptance/bundle/deployment/bind/pipelines/update/out.bind-success.direct.txt b/acceptance/bundle/deployment/bind/pipelines/update/out.bind-success.direct.txt new file mode 100644 index 0000000000..6c88a3134b --- /dev/null +++ b/acceptance/bundle/deployment/bind/pipelines/update/out.bind-success.direct.txt @@ -0,0 +1,5 @@ + +>>> [CLI] bundle deployment bind foo [NEW_PIPELINE_ID] --auto-approve +Updating deployment state... +Successfully bound pipeline with an id '[NEW_PIPELINE_ID]' +Run 'bundle deploy' to deploy changes to your workspace diff --git a/acceptance/bundle/deployment/bind/pipelines/recreate/out.deploy.requests.terraform.json b/acceptance/bundle/deployment/bind/pipelines/update/out.deploy.requests.direct.json similarity index 75% rename from acceptance/bundle/deployment/bind/pipelines/recreate/out.deploy.requests.terraform.json rename to acceptance/bundle/deployment/bind/pipelines/update/out.deploy.requests.direct.json index ce3ee109e0..78f256ba44 100644 --- a/acceptance/bundle/deployment/bind/pipelines/recreate/out.deploy.requests.terraform.json +++ b/acceptance/bundle/deployment/bind/pipelines/update/out.deploy.requests.direct.json @@ -6,12 +6,15 @@ } } { - "method": "DELETE", - "path": "/api/2.0/pipelines/[NEW_PIPELINE_ID]" + "method": "POST", + "path": "/api/2.0/workspace/mkdirs", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-recreate/default/files" + } } { - "method": "POST", - "path": "/api/2.0/pipelines", + "method": "PUT", + "path": "/api/2.0/pipelines/[NEW_PIPELINE_ID]", "body": { "catalog": "main", "channel": "CURRENT", diff --git a/acceptance/bundle/deployment/bind/pipelines/recreate/out.summary.terraform.json b/acceptance/bundle/deployment/bind/pipelines/update/out.summary.json similarity index 100% rename from acceptance/bundle/deployment/bind/pipelines/recreate/out.summary.terraform.json rename to acceptance/bundle/deployment/bind/pipelines/update/out.summary.json diff --git a/acceptance/bundle/deployment/bind/pipelines/update/out.summary.terraform.json b/acceptance/bundle/deployment/bind/pipelines/update/out.summary.terraform.json deleted file mode 100644 index d4d9b72a85..0000000000 --- a/acceptance/bundle/deployment/bind/pipelines/update/out.summary.terraform.json +++ /dev/null @@ -1,51 +0,0 @@ -{ - "bundle": { - "environment": "default", - "git": { - "bundle_root_path": "." - }, - "name": "test-pipeline-recreate", - "target": "default" - }, - "resources": { - "pipelines": { - "foo": { - "catalog": "main", - "channel": "CURRENT", - "deployment": { - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-recreate/default/state/metadata.json" - }, - "edition": "ADVANCED", - "id": "[NEW_PIPELINE_ID]", - "libraries": [ - { - "notebook": { - "path": "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-recreate/default/files/nb" - } - } - ], - "name": "test-pipeline", - "url": "[DATABRICKS_URL]/pipelines/[NEW_PIPELINE_ID]?o=[NUMID]" - } - } - }, - "sync": { - "paths": [ - "." - ] - }, - "workspace": { - "artifact_path": "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-recreate/default/artifacts", - "current_user": { - "domain_friendly_name": "[USERNAME]", - "id": "[USERID]", - "short_name": "[USERNAME]", - "userName": "[USERNAME]" - }, - "file_path": "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-recreate/default/files", - "resource_path": "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-recreate/default/resources", - "root_path": "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-recreate/default", - "state_path": "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-recreate/default/state" - } -} diff --git a/acceptance/bundle/deployment/bind/pipelines/update/out.test.toml b/acceptance/bundle/deployment/bind/pipelines/update/out.test.toml index 90061dedb1..d560f1de04 100644 --- a/acceptance/bundle/deployment/bind/pipelines/update/out.test.toml +++ b/acceptance/bundle/deployment/bind/pipelines/update/out.test.toml @@ -2,4 +2,4 @@ Local = true Cloud = false [EnvMatrix] - DATABRICKS_BUNDLE_ENGINE = ["terraform"] + DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/pipelines/update/script b/acceptance/bundle/deployment/bind/pipelines/update/script index 1235b895d6..5d2e487f10 100644 --- a/acceptance/bundle/deployment/bind/pipelines/update/script +++ b/acceptance/bundle/deployment/bind/pipelines/update/script @@ -9,7 +9,7 @@ rm -f out.requests.txt trace $CLI bundle deployment bind foo $NEW_PIPELINE_ID --auto-approve &> out.bind-success.$DATABRICKS_BUNDLE_ENGINE.txt print_requests.py '^//import-file/' '^//workspace/delete' -trace $CLI bundle summary -o json > out.summary.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle summary -o json > out.summary.json trace $CLI bundle plan rm -f out.requests.txt diff --git a/acceptance/bundle/deployment/bind/quality-monitor/out.test.toml b/acceptance/bundle/deployment/bind/quality-monitor/out.test.toml index 90061dedb1..d560f1de04 100644 --- a/acceptance/bundle/deployment/bind/quality-monitor/out.test.toml +++ b/acceptance/bundle/deployment/bind/quality-monitor/out.test.toml @@ -2,4 +2,4 @@ Local = true Cloud = false [EnvMatrix] - DATABRICKS_BUNDLE_ENGINE = ["terraform"] + DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/registered-model/out.test.toml b/acceptance/bundle/deployment/bind/registered-model/out.test.toml index 9016731b25..d61c11e25c 100644 --- a/acceptance/bundle/deployment/bind/registered-model/out.test.toml +++ b/acceptance/bundle/deployment/bind/registered-model/out.test.toml @@ -3,4 +3,4 @@ Cloud = true RequiresUnityCatalog = true [EnvMatrix] - DATABRICKS_BUNDLE_ENGINE = ["terraform"] + DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/schema/out.test.toml b/acceptance/bundle/deployment/bind/schema/out.test.toml index 9016731b25..d61c11e25c 100644 --- a/acceptance/bundle/deployment/bind/schema/out.test.toml +++ b/acceptance/bundle/deployment/bind/schema/out.test.toml @@ -3,4 +3,4 @@ Cloud = true RequiresUnityCatalog = true [EnvMatrix] - DATABRICKS_BUNDLE_ENGINE = ["terraform"] + DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/secret-scope/out.test.toml b/acceptance/bundle/deployment/bind/secret-scope/out.test.toml index 9016731b25..d61c11e25c 100644 --- a/acceptance/bundle/deployment/bind/secret-scope/out.test.toml +++ b/acceptance/bundle/deployment/bind/secret-scope/out.test.toml @@ -3,4 +3,4 @@ Cloud = true RequiresUnityCatalog = true [EnvMatrix] - DATABRICKS_BUNDLE_ENGINE = ["terraform"] + DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/sql_warehouse/out.test.toml b/acceptance/bundle/deployment/bind/sql_warehouse/out.test.toml index 90061dedb1..d560f1de04 100644 --- a/acceptance/bundle/deployment/bind/sql_warehouse/out.test.toml +++ b/acceptance/bundle/deployment/bind/sql_warehouse/out.test.toml @@ -2,4 +2,4 @@ Local = true Cloud = false [EnvMatrix] - DATABRICKS_BUNDLE_ENGINE = ["terraform"] + DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/volume/out.test.toml b/acceptance/bundle/deployment/bind/volume/out.test.toml index 9016731b25..d61c11e25c 100644 --- a/acceptance/bundle/deployment/bind/volume/out.test.toml +++ b/acceptance/bundle/deployment/bind/volume/out.test.toml @@ -3,4 +3,4 @@ Cloud = true RequiresUnityCatalog = true [EnvMatrix] - DATABRICKS_BUNDLE_ENGINE = ["terraform"] + DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/test.toml b/acceptance/bundle/deployment/test.toml index 111f94f088..c7c6f58ed6 100644 --- a/acceptance/bundle/deployment/test.toml +++ b/acceptance/bundle/deployment/test.toml @@ -1,3 +1 @@ Cloud = true - -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] # bind,unbind not implemented diff --git a/acceptance/bundle/deployment/unbind/grants/out.test.toml b/acceptance/bundle/deployment/unbind/grants/out.test.toml index 8409922737..d61c11e25c 100644 --- a/acceptance/bundle/deployment/unbind/grants/out.test.toml +++ b/acceptance/bundle/deployment/unbind/grants/out.test.toml @@ -3,5 +3,4 @@ Cloud = true RequiresUnityCatalog = true [EnvMatrix] - DATABRICKS_BUNDLE_ENGINE = ["terraform"] - Ignore = [".databricks"] + DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/unbind/grants/test.toml b/acceptance/bundle/deployment/unbind/grants/test.toml index ca6de773cd..7eb723bd30 100644 --- a/acceptance/bundle/deployment/unbind/grants/test.toml +++ b/acceptance/bundle/deployment/unbind/grants/test.toml @@ -1,8 +1,5 @@ RequiresUnityCatalog = true -[EnvMatrix] - DATABRICKS_BUNDLE_ENGINE = ["terraform"] - Ignore = [ ".databricks", ] diff --git a/acceptance/bundle/deployment/unbind/job/out.test.toml b/acceptance/bundle/deployment/unbind/job/out.test.toml index 90061dedb1..d560f1de04 100644 --- a/acceptance/bundle/deployment/unbind/job/out.test.toml +++ b/acceptance/bundle/deployment/unbind/job/out.test.toml @@ -2,4 +2,4 @@ Local = true Cloud = false [EnvMatrix] - DATABRICKS_BUNDLE_ENGINE = ["terraform"] + DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/unbind/permissions/out.test.toml b/acceptance/bundle/deployment/unbind/permissions/out.test.toml index 4dccab7df1..01ed6822af 100644 --- a/acceptance/bundle/deployment/unbind/permissions/out.test.toml +++ b/acceptance/bundle/deployment/unbind/permissions/out.test.toml @@ -2,5 +2,4 @@ Local = true Cloud = true [EnvMatrix] - DATABRICKS_BUNDLE_ENGINE = ["terraform"] - Ignore = [".databricks"] + DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/unbind/permissions/test.toml b/acceptance/bundle/deployment/unbind/permissions/test.toml index 0708e106d0..bd4b66fe75 100644 --- a/acceptance/bundle/deployment/unbind/permissions/test.toml +++ b/acceptance/bundle/deployment/unbind/permissions/test.toml @@ -1,6 +1,3 @@ -[EnvMatrix] - DATABRICKS_BUNDLE_ENGINE = ["terraform"] - Ignore = [ ".databricks", ] diff --git a/acceptance/bundle/deployment/unbind/python-job/out.test.toml b/acceptance/bundle/deployment/unbind/python-job/out.test.toml index 90061dedb1..d560f1de04 100644 --- a/acceptance/bundle/deployment/unbind/python-job/out.test.toml +++ b/acceptance/bundle/deployment/unbind/python-job/out.test.toml @@ -2,4 +2,4 @@ Local = true Cloud = false [EnvMatrix] - DATABRICKS_BUNDLE_ENGINE = ["terraform"] + DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/bundle/direct/bind.go b/bundle/direct/bind.go new file mode 100644 index 0000000000..74951b1d3e --- /dev/null +++ b/bundle/direct/bind.go @@ -0,0 +1,220 @@ +package direct + +import ( + "context" + "fmt" + "os" + "strings" + + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/bundle/direct/dstate" + "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/structs/structaccess" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/databricks-sdk-go" +) + +// ErrResourceAlreadyBound is returned when attempting to bind a resource +// that is already bound to a different ID in the state. +type ErrResourceAlreadyBound struct { + ResourceKey string + ExistingID string + NewID string +} + +func (e ErrResourceAlreadyBound) Error() string { + if e.ExistingID != e.NewID { + return fmt.Sprintf("Resource already managed\n\n"+ + "The bundle is already managing a resource for %s with ID '%s'.\n"+ + "To bind to a different resource with ID '%s', you must first unbind the existing resource.", + e.ResourceKey, e.ExistingID, e.NewID) + } + return fmt.Sprintf("Resource already managed\n\nThe bundle is already managing resource for %s and it is bound to the requested ID %s.", + e.ResourceKey, e.ExistingID) +} + +// BindResult contains the result of a bind operation including any detected changes. +type BindResult struct { + // HasChanges is true if deploying after bind would make changes to the resource + HasChanges bool + // Action is the planned action for the bound resource (e.g., "skip", "update", "recreate") + Action deployplan.ActionType + // Plan contains the full deployment plan for the bound resource + Plan *deployplan.Plan + // TempStatePath is the path to the temporary state file + TempStatePath string + // StatePath is the path to the final state file + StatePath string +} + +// Bind adds an existing workspace resource to a temporary state and calculates +// if there will be any changes when deploying. +// +// Flow: +// 1. Load current state file +// 2. Update id of the target resource +// 3. Update state path to temp file (state file + ".temp-bind") +// 4. Perform plan + update to populate state with resolved config +// 5. Calculate plan again - this is the plan presented to the user +// +// Call Finalize to commit the state or Cancel to discard. +func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.WorkspaceClient, configRoot *config.Root, statePath, resourceKey, resourceID string) (*BindResult, error) { + // Check if the resource is already managed (bound to a different ID) + var checkStateDB dstate.DeploymentState + if err := checkStateDB.Open(statePath); err == nil { + if existing, ok := checkStateDB.GetResourceEntry(resourceKey); ok { + return nil, ErrResourceAlreadyBound{ + ResourceKey: resourceKey, + ExistingID: existing.ID, + NewID: resourceID, + } + } + } + + tmpStatePath := statePath + ".temp-bind" + + // Copy existing state to temp location if it exists + if data, err := os.ReadFile(statePath); err == nil { + if err := os.WriteFile(tmpStatePath, data, 0o600); err != nil { + return nil, err + } + } + + // Open temp state + err := b.StateDB.Open(tmpStatePath) + if err != nil { + os.Remove(tmpStatePath) + return nil, err + } + + // Save state with ID and empty state (like migrate does) + err = b.StateDB.SaveState(resourceKey, resourceID, struct{}{}, nil) + if err != nil { + os.Remove(tmpStatePath) + return nil, err + } + + // Finalize to write temp state to disk so CalculatePlan can read it + err = b.StateDB.Finalize() + if err != nil { + os.Remove(tmpStatePath) + return nil, err + } + + log.Infof(ctx, "Bound %s to id=%s (in temp state)", resourceKey, resourceID) + + // First plan + update: populate state with resolved config + plan, err := b.CalculatePlan(ctx, client, configRoot, tmpStatePath) + if err != nil { + os.Remove(tmpStatePath) + return nil, err + } + + // Populate the state with the resolved config + entry := plan.Plan[resourceKey] + sv, ok := b.StructVarCache.Load(resourceKey) + if ok && sv != nil { + var dependsOn []deployplan.DependsOnEntry + if entry != nil { + dependsOn = entry.DependsOn + } + + // Copy etag from remote state for dashboards. + // Dashboards store "etag" in state which is not provided by user but comes from remote. + // If we don't store "etag" in state, we won't detect remote drift correctly. + if strings.Contains(resourceKey, ".dashboards.") && entry != nil && entry.RemoteState != nil { + etag, err := structaccess.Get(entry.RemoteState, structpath.NewStringKey(nil, "etag")) + if err == nil && etag != nil { + if etagStr, ok := etag.(string); ok && etagStr != "" { + _ = structaccess.Set(sv.Value, structpath.NewStringKey(nil, "etag"), etagStr) + } + } + } + + err = b.StateDB.SaveState(resourceKey, resourceID, sv.Value, dependsOn) + if err != nil { + os.Remove(tmpStatePath) + return nil, err + } + + err = b.StateDB.Finalize() + if err != nil { + os.Remove(tmpStatePath) + return nil, err + } + } + + // Second plan: this is the plan to present to the user (change between remote resource and config) + plan, err = b.CalculatePlan(ctx, client, configRoot, tmpStatePath) + if err != nil { + os.Remove(tmpStatePath) + return nil, err + } + + // Check if the bound resource has changes + result := &BindResult{ + HasChanges: false, + Action: deployplan.Skip, + Plan: plan, + TempStatePath: tmpStatePath, + StatePath: statePath, + } + + entry = plan.Plan[resourceKey] + if entry != nil { + result.Action = entry.Action + result.HasChanges = result.Action != deployplan.Skip && result.Action != deployplan.Undefined + } + + return result, nil +} + +// Finalize completes the bind operation by renaming the temp state to the final location. +func (result *BindResult) Finalize() error { + if result == nil || result.TempStatePath == "" { + return nil + } + return os.Rename(result.TempStatePath, result.StatePath) +} + +// Cancel cleans up temp state without committing changes. +func (result *BindResult) Cancel() { + if result != nil && result.TempStatePath != "" { + os.Remove(result.TempStatePath) + } +} + +// Unbind removes a resource from direct engine state without deleting +// the workspace resource. Also removes associated permissions/grants entries. +func (b *DeploymentBundle) Unbind(ctx context.Context, statePath, resourceKey string) error { + err := b.StateDB.Open(statePath) + if err != nil { + return err + } + + // Delete the main resource + err = b.StateDB.DeleteState(resourceKey) + if err != nil { + return err + } + + log.Infof(ctx, "Unbound %s", resourceKey) + + // Also delete associated permissions/grants if they exist + // These are stored as "resources.jobs.foo.permissions" or "resources.schemas.bar.grants" + permissionsKey := resourceKey + ".permissions" + grantsKey := resourceKey + ".grants" + + for key := range b.StateDB.Data.State { + if key == permissionsKey || key == grantsKey || strings.HasPrefix(key, resourceKey+".") { + err = b.StateDB.DeleteState(key) + if err != nil { + return err + } + log.Infof(ctx, "Unbound %s", key) + } + } + + return b.StateDB.Finalize() +} diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index c369195cf2..c105e9c49c 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "os" + "path/filepath" "strings" "sync" @@ -177,6 +178,12 @@ func (db *DeploymentState) unlockedSave() error { return err } + // Create parent directories if they don't exist + dir := filepath.Dir(db.Path) + if err := os.MkdirAll(dir, 0o755); err != nil { + return fmt.Errorf("failed to create directory %#v: %w", dir, err) + } + err = os.WriteFile(db.Path, data, 0o600) if err != nil { return fmt.Errorf("failed to save resources state to %#v: %w", db.Path, err) diff --git a/bundle/phases/bind.go b/bundle/phases/bind.go index aaf329480d..715f97f528 100644 --- a/bundle/phases/bind.go +++ b/bundle/phases/bind.go @@ -2,14 +2,20 @@ package phases import ( "context" + "encoding/json" + "errors" + "fmt" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config/engine" "github.com/databricks/cli/bundle/deploy/lock" "github.com/databricks/cli/bundle/deploy/terraform" + "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/bundle/statemgmt" + "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/cli/libs/utils" ) func Bind(ctx context.Context, b *bundle.Bundle, opts *terraform.BindOptions) { @@ -30,18 +36,87 @@ func Bind(ctx context.Context, b *bundle.Bundle, opts *terraform.BindOptions) { bundle.ApplyContext(ctx, b, lock.Release(lock.GoalBind)) }() - bundle.ApplySeqContext(ctx, b, - terraform.Interpolate(), - terraform.Write(), - terraform.Import(opts), - ) - if logdiag.HasError(ctx) { - return + if engine.IsDirect() { + // Direct engine: import into temp state, run plan, check for changes + // This follows the same pattern as terraform import + groupName := terraform.TerraformToGroupName[opts.ResourceType] + resourceKey := fmt.Sprintf("resources.%s.%s", groupName, opts.ResourceKey) + _, statePath := b.StateFilenameDirect(ctx) + + result, err := b.DeploymentBundle.Bind(ctx, b.WorkspaceClient(), &b.Config, statePath, resourceKey, opts.ResourceId) + if err != nil { + logdiag.LogError(ctx, err) + return + } + + // If there are changes and auto-approve is not set, show plan and ask for confirmation + if result.HasChanges && !opts.AutoApprove { + // Display the planned changes for the bound resource + cmdio.LogString(ctx, fmt.Sprintf("Plan: %s %s", result.Action, resourceKey)) + + // Show details of what will change + if result.Plan != nil { + if entry, ok := result.Plan.Plan[resourceKey]; ok && entry != nil && len(entry.Changes) > 0 { + cmdio.LogString(ctx, "\nChanges detected:") + for _, field := range utils.SortedKeys(entry.Changes) { + change := entry.Changes[field] + if change.Action != deployplan.Skip { + cmdio.LogString(ctx, fmt.Sprintf(" ~ %s: %v -> %v", field, jsonDump(ctx, change.Remote, field), jsonDump(ctx, change.New, field))) + } + } + cmdio.LogString(ctx, "") + } + } + + if !cmdio.IsPromptSupported(ctx) { + result.Cancel() + logdiag.LogError(ctx, errors.New("This bind operation requires user confirmation, but the current console does not support prompting. Please specify --auto-approve if you would like to skip prompts and proceed.")) //nolint + return + } + + ans, err := cmdio.AskYesOrNo(ctx, "Confirm import changes? Changes will be remotely applied only after running 'bundle deploy'.") + if err != nil { + result.Cancel() + logdiag.LogError(ctx, err) + return + } + if !ans { + result.Cancel() + logdiag.LogError(ctx, errors.New("import aborted")) + return + } + } + + // Finalize: rename temp state to final location + err = result.Finalize() + if err != nil { + logdiag.LogError(ctx, err) + return + } + } else { + // Terraform engine: use terraform import + bundle.ApplySeqContext(ctx, b, + terraform.Interpolate(), + terraform.Write(), + terraform.Import(opts), + ) + if logdiag.HasError(ctx) { + return + } } statemgmt.PushResourcesState(ctx, b, engine) } +func jsonDump(ctx context.Context, v any, field string) string { + b, err := json.Marshal(v) + if err != nil { + log.Warnf(ctx, "Cannot marshal %s: %s", field, err) + return "??" + } + return string(b) +} + func Unbind(ctx context.Context, b *bundle.Bundle, bundleType, tfResourceType, resourceKey string) { log.Info(ctx, "Phase: unbind") @@ -60,13 +135,24 @@ func Unbind(ctx context.Context, b *bundle.Bundle, bundleType, tfResourceType, r bundle.ApplyContext(ctx, b, lock.Release(lock.GoalUnbind)) }() - bundle.ApplySeqContext(ctx, b, - terraform.Interpolate(), - terraform.Write(), - terraform.Unbind(bundleType, tfResourceType, resourceKey), - ) - if logdiag.HasError(ctx) { - return + if engine.IsDirect() { + groupName := terraform.TerraformToGroupName[tfResourceType] + fullResourceKey := fmt.Sprintf("resources.%s.%s", groupName, resourceKey) + _, statePath := b.StateFilenameDirect(ctx) + err := b.DeploymentBundle.Unbind(ctx, statePath, fullResourceKey) + if err != nil { + logdiag.LogError(ctx, err) + return + } + } else { + bundle.ApplySeqContext(ctx, b, + terraform.Interpolate(), + terraform.Write(), + terraform.Unbind(bundleType, tfResourceType, resourceKey), + ) + if logdiag.HasError(ctx) { + return + } } statemgmt.PushResourcesState(ctx, b, engine)