Add code_source block in ai_runtime_task to provide cleaner UX

.nextchanges/bundles/ai-runtime-code-source-block.md

@@ -0,0 +1 @@
+Add a DABs-native `code_source` block to `ai_runtime_task`, matching the AIR CLI's `train.yaml`. Set `code_source: { root_path, include_paths, git }` on the task and `bundle deploy` packages the directory into a content-addressed tarball (honoring `.gitignore` and sync include/exclude, narrowing to `include_paths`, or archiving a pinned `git` branch/commit), uploads it, and sets `code_source_path` for you. The existing string `code_source_path: ./dir` form is unchanged; the two are mutually exclusive on a task.

acceptance/bundle/ai_runtime_task/code_source/block.yml

@@ -0,0 +1,28 @@
+bundle:
+  name: ai-runtime-block
+
+resources:
+  jobs:
+    train:
+      name: "[${bundle.target}] AI Runtime training"
+      tasks:
+        # code_source block, root_path form: packages ./src exactly like a bare
+        # code_source_path (content-addressed tarball under .air_snapshots, uploaded by
+        # file sync, code_source_path rewritten to the uploaded workspace path).
+        - task_key: train
+          environment_key: default
+          ai_runtime_task:
+            experiment: my-training
+            code_source:
+              root_path: ./src
+            deployments:
+              - command_path: src/command.sh
+                compute:
+                  accelerator_type: GPU_8xH100
+                  accelerator_count: 8
+      environments:
+        - environment_key: default
+          spec:
+            environment_version: "5"
+            dependencies:
+              - torch>=2.0.0

acceptance/bundle/ai_runtime_task/code_source/git.yml

@@ -0,0 +1,28 @@
+bundle:
+  name: ai-runtime-git
+
+resources:
+  jobs:
+    train:
+      name: "[${bundle.target}] AI Runtime training"
+      tasks:
+        # code_source.git pins the snapshot to a committed revision: the archive is
+        # produced with `git archive` of that commit, not the working tree. COMMIT_SHA
+        # is substituted with the repo HEAD by the script.
+        - task_key: train
+          environment_key: default
+          ai_runtime_task:
+            experiment: my-training
+            code_source:
+              root_path: ./src
+              git:
+                commit: COMMIT_SHA
+            deployments:
+              - command_path: src/command.sh
+                compute:
+                  accelerator_type: GPU_1xA10
+                  accelerator_count: 1
+      environments:
+        - environment_key: default
+          spec:
+            environment_version: "5"

acceptance/bundle/ai_runtime_task/code_source/include_paths.yml

@@ -0,0 +1,28 @@
+bundle:
+  name: ai-runtime-includes
+
+resources:
+  jobs:
+    train:
+      name: "[${bundle.target}] AI Runtime training"
+      tasks:
+        # include_paths narrows the packaged archive to a subset of root_path: only
+        # keep/ and configs/ are archived; drop/ is left out.
+        - task_key: train
+          environment_key: default
+          ai_runtime_task:
+            experiment: my-training
+            code_source:
+              root_path: ./src
+              include_paths:
+                - keep
+                - configs
+            deployments:
+              - command_path: src/keep/command.sh
+                compute:
+                  accelerator_type: GPU_1xA10
+                  accelerator_count: 1
+      environments:
+        - environment_key: default
+          spec:
+            environment_version: "5"

acceptance/bundle/ai_runtime_task/code_source/output.txt

@@ -0,0 +1,181 @@
+
+=== root_path form packages ./src like a bare code_source_path
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/ai-runtime-block/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> list_code_snapshot.py
+# .air_snapshots/src_[SNAPSHOT].tar.gz
+src/command.sh
+src/configs/model.yaml
+src/drop/ignored.py
+src/keep/command.sh
+src/keep/train.py
+src/train.py
+
+>>> print_requests.py --sort --del-field raw_body //.air_snapshots/ //jobs/create
+{
+  "method": "POST",
+  "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/ai-runtime-block/default/files/.air_snapshots/src_[SNAPSHOT].tar.gz",
+  "q": {
+    "overwrite": "true"
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.2/jobs/create",
+  "body": {
+    "deployment": {
+      "kind": "BUNDLE",
+      "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/ai-runtime-block/default/state/metadata.json"
+    },
+    "edit_mode": "UI_LOCKED",
+    "environments": [
+      {
+        "environment_key": "default",
+        "spec": {
+          "dependencies": [
+            "torch>=2.0.0"
+          ],
+          "environment_version": "5"
+        }
+      }
+    ],
+    "format": "MULTI_TASK",
+    "max_concurrent_runs": 1,
+    "name": "[default] AI Runtime training",
+    "queue": {
+      "enabled": true
+    },
+    "tasks": [
+      {
+        "ai_runtime_task": {
+          "code_source_path": "/Workspace/Users/[USERNAME]/.bundle/ai-runtime-block/default/files/.air_snapshots/src_[SNAPSHOT].tar.gz",
+          "deployments": [
+            {
+              "command_path": "/Workspace/Users/[USERNAME]/.bundle/ai-runtime-block/default/files/src/command.sh",
+              "compute": {
+                "accelerator_count": 8,
+                "accelerator_type": "GPU_8xH100"
+              }
+            }
+          ],
+          "experiment": "my-training"
+        },
+        "environment_key": "default",
+        "task_key": "train"
+      }
+    ]
+  }
+}
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.jobs.train
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/ai-runtime-block/default
+
+Deleting files...
+Destroy complete!
+
+=== include_paths narrows the archive to keep/ and configs/ (drop/ excluded)
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/ai-runtime-includes/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> list_code_snapshot.py
+# .air_snapshots/src_[SNAPSHOT].tar.gz
+src/configs/model.yaml
+src/keep/command.sh
+src/keep/train.py
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.jobs.train
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/ai-runtime-includes/default
+
+Deleting files...
+Destroy complete!
+
+=== git.commit archives the pinned revision via git archive
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/ai-runtime-git/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> list_code_snapshot.py
+# .air_snapshots/src_[SNAPSHOT].tar.gz
+src
+src/command.sh
+src/configs
+src/configs/model.yaml
+src/drop
+src/drop/ignored.py
+src/keep
+src/keep/command.sh
+src/keep/train.py
+src/train.py
+
+>>> print_requests.py --sort --del-field raw_body //jobs/create
+{
+  "method": "POST",
+  "path": "/api/2.2/jobs/create",
+  "body": {
+    "deployment": {
+      "kind": "BUNDLE",
+      "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/ai-runtime-git/default/state/metadata.json"
+    },
+    "edit_mode": "UI_LOCKED",
+    "environments": [
+      {
+        "environment_key": "default",
+        "spec": {
+          "environment_version": "5"
+        }
+      }
+    ],
+    "format": "MULTI_TASK",
+    "max_concurrent_runs": 1,
+    "name": "[default] AI Runtime training",
+    "queue": {
+      "enabled": true
+    },
+    "tasks": [
+      {
+        "ai_runtime_task": {
+          "code_source_path": "/Workspace/Users/[USERNAME]/.bundle/ai-runtime-git/default/files/.air_snapshots/src_[SNAPSHOT].tar.gz",
+          "deployments": [
+            {
+              "command_path": "/Workspace/Users/[USERNAME]/.bundle/ai-runtime-git/default/files/src/command.sh",
+              "compute": {
+                "accelerator_count": 1,
+                "accelerator_type": "GPU_1xA10"
+              }
+            }
+          ],
+          "experiment": "my-training"
+        },
+        "environment_key": "default",
+        "task_key": "train"
+      }
+    ]
+  }
+}
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.jobs.train
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/ai-runtime-git/default
+
+Deleting files...
+Destroy complete!

acceptance/bundle/ai_runtime_task/code_source/script

@@ -0,0 +1,38 @@
+# The DABs-native ai_runtime_task.code_source block, exercised end-to-end in three
+# forms against a shared ./src tree. Each fixture is copied to databricks.yml, deployed
+# (packaging + upload + code_source_path rewrite), inspected, then destroyed. The git
+# form runs last because it needs a git repo; the working-tree forms must run before
+# git-repo-init so they exercise the non-git filesystem walk.
+
+title "root_path form packages ./src like a bare code_source_path\n"
+cp block.yml databricks.yml
+trace $CLI bundle deploy
+trace list_code_snapshot.py
+trace print_requests.py --sort --del-field raw_body '//.air_snapshots/' '//jobs/create'
+trace $CLI bundle destroy --auto-approve
+rm out.requests.txt
+
+title "include_paths narrows the archive to keep/ and configs/ (drop/ excluded)\n"
+cp include_paths.yml databricks.yml
+trace $CLI bundle deploy
+trace list_code_snapshot.py
+trace $CLI bundle destroy --auto-approve
+rm out.requests.txt
+
+title "git.commit archives the pinned revision via git archive\n"
+# Pinned commit dates keep the resolved HEAD SHA stable across runs.
+export GIT_AUTHOR_DATE="2020-01-01T00:00:00Z"
+export GIT_COMMITTER_DATE="2020-01-01T00:00:00Z"
+git-repo-init
+git add src
+git commit -qm 'Add src'
+# git.commit tolerates a dirty working tree (the pinned revision is archived), which
+# matters here because the harness writes output.txt into the working dir as it runs.
+cp git.yml databricks.yml
+update_file.py databricks.yml COMMIT_SHA "$(git rev-parse HEAD)"
+trace $CLI bundle deploy
+trace list_code_snapshot.py
+trace print_requests.py --sort --del-field raw_body '//jobs/create'
+trace $CLI bundle destroy --auto-approve
+
+rm -fr .git out.requests.txt databricks.yml

acceptance/bundle/ai_runtime_task/code_source/src/command.sh

@@ -0,0 +1,2 @@
+#!/bin/bash
+python train.py

acceptance/bundle/ai_runtime_task/code_source/src/configs/model.yaml

@@ -0,0 +1 @@
+lr: 0.001

acceptance/bundle/ai_runtime_task/code_source/src/drop/ignored.py

@@ -0,0 +1 @@
+print("this file must not be packaged")

acceptance/bundle/ai_runtime_task/code_source/src/keep/command.sh

@@ -0,0 +1,2 @@
+#!/bin/bash
+python train.py

acceptance/bundle/ai_runtime_task/code_source/src/keep/train.py

@@ -0,0 +1 @@
+print("training")

acceptance/bundle/ai_runtime_task/code_source/src/train.py

@@ -0,0 +1 @@
+print("training")

acceptance/bundle/ai_runtime_task/code_source/test.toml

@@ -0,0 +1,13 @@
+RecordRequests = true
+
+Ignore = [
+    '.databricks',
+    'databricks.yml',
+]
+
+# The archive is content-addressed: <dir>_<sha256[:16]>.tar.gz. Collapse the hash to a
+# token so the test does not pin a specific digest (git's gzip output is also not
+# byte-identical across git versions).
+[[Repls]]
+Old = 'src_[0-9a-f]{16}\.tar\.gz'
+New = 'src_[SNAPSHOT].tar.gz'

acceptance/bundle/ai_runtime_task/code_source_errors/conflict.yml

@@ -0,0 +1,24 @@
+bundle:
+  name: ai-runtime-conflict
+
+resources:
+  jobs:
+    train:
+      tasks:
+        - task_key: train
+          environment_key: default
+          ai_runtime_task:
+            experiment: my-training
+            # Both forms set on one task: mutually exclusive, rejected at validate.
+            code_source_path: ./src
+            code_source:
+              root_path: ./src
+            deployments:
+              - command_path: src/command.sh
+                compute:
+                  accelerator_type: GPU_1xA10
+                  accelerator_count: 1
+      environments:
+        - environment_key: default
+          spec:
+            environment_version: "5"