feat(ci): reap stale CI workspaces and template versions on staging

.github/workflows/ci-reap.yml

@@ -0,0 +1,70 @@
+name: Reap stale CI workspaces
+
+# Decoupled janitor for staging-coder.ddev.com.
+#
+# Integration-test runs push template versions and create workspaces (owned by
+# ci-bot) on staging. Their per-run cleanup runs in `if: always()` steps inside
+# the test job, which never reach the server when a run is cancelled
+# (cancel-in-progress on a new push), force killed, or its workspace lands in
+# Failed. This workflow reaps by owner + state + age independently of any test
+# run, so an orphan never outlives one janitor interval, and merged / closed /
+# abandoned PRs get cleaned up even though no replacement run ever comes.
+#
+# Requires (same as the integration-test workflows):
+#   Repository variable: TEST_CODER_URL           - https://staging-coder.ddev.com
+#   Repository secret:   OP_SERVICE_ACCOUNT_TOKEN - 1Password service account
+#   1Password item:      op://test-secrets/TEST_CODER_SESSION_TOKEN/credential
+
+env:
+  FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
+
+# Never run two janitors at once; let an in-flight one finish rather than
+# cancelling it mid-delete.
+concurrency:
+  group: ci-reap
+  cancel-in-progress: false
+
+on:
+  schedule:
+    - cron: '*/15 * * * *'
+  workflow_dispatch:
+    inputs:
+      age_minutes:
+        description: 'Reap running ci-bot workspaces older than this many minutes'
+        type: string
+        required: false
+        default: '20'
+      dry_run:
+        description: 'Dry run (list only, delete nothing)'
+        type: boolean
+        required: false
+        default: false
+
+jobs:
+  reap:
+    name: Reap stale CI workspaces and versions
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash -euo pipefail {0}
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Load 1Password secrets
+        uses: 1password/load-secrets-action@v4
+        with:
+          export-env: true
+        env:
+          OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
+          TEST_CODER_SESSION_TOKEN: "op://test-secrets/TEST_CODER_SESSION_TOKEN/credential"
+
+      - uses: coder/setup-action@v1
+        with:
+          access_url: ${{ vars.TEST_CODER_URL }}
+          coder_session_token: ${{ env.TEST_CODER_SESSION_TOKEN }}
+
+      - name: Reap stale CI workspaces and template versions
+        env:
+          AGE_MINUTES: ${{ github.event.inputs.age_minutes || '20' }}
+          DRY_RUN: ${{ github.event.inputs.dry_run || 'false' }}
+        run: ./scripts/ci-reap-staging.sh $([[ "$DRY_RUN" == "true" ]] || echo --force)

.github/workflows/drupal-contrib-integration-test.yml

@@ -391,14 +391,10 @@ jobs:
     if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login == github.repository_owner }}
     runs-on: ubuntu-latest
     strategy:
+      # PRs run a single module (smoke); push-to-main and nightly run the full
+      # set. Cuts per-PR workspace load on staging. See ci-reap.yml.
       matrix:
-        include:
-          - project: token
-            drupal_version: "11"
-          - project: pathauto
-            drupal_version: "11"
-          - project: skipto
-            drupal_version: "12"
+        include: ${{ github.event_name == 'pull_request' && fromJSON('[{"project":"token","drupal_version":"11"}]') || fromJSON('[{"project":"token","drupal_version":"11"},{"project":"pathauto","drupal_version":"11"},{"project":"skipto","drupal_version":"12"}]') }}
       fail-fast: false
     defaults:
       run:
@@ -438,6 +434,9 @@ jobs:
             --variable workspace_image_registry=index.docker.io/ddev/coder-ddev \
             --variable github_token=${{ secrets.GITHUB_TOKEN }}
 
+      - name: Reap predecessor workspaces (cancelled prior runs of this cell)
+        run: ./scripts/ci-reap-family.sh "gc-${{ matrix.project }}-d${{ matrix.drupal_version }}-" "${{ env.WORKSPACE_NAME }}"
+
       - name: Create workspace
         run: |
           coder create ${{ env.WORKSPACE_NAME }} \
@@ -590,6 +589,9 @@ jobs:
             --variable workspace_image_registry=index.docker.io/ddev/coder-ddev \
             --variable github_token=${{ secrets.GITHUB_TOKEN }}
 
+      - name: Reap predecessor workspaces (cancelled prior runs of this cell)
+        run: ./scripts/ci-reap-family.sh "gc-fork-" "${{ env.WORKSPACE_NAME }}"
+
       - name: Create workspace
         run: |
           coder create ${{ env.WORKSPACE_NAME }} \

.github/workflows/drupal-integration-test.yml

@@ -201,8 +201,10 @@ jobs:
     if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login == github.repository_owner }}
     runs-on: ubuntu-latest
     strategy:
+      # PRs run a single Drupal version (smoke); push-to-main and nightly run the
+      # full matrix. Cuts per-PR workspace load on staging. See ci-reap.yml.
       matrix:
-        drupal_version: ["12", "11"]
+        drupal_version: ${{ github.event_name == 'pull_request' && fromJSON('["12"]') || fromJSON('["12", "11"]') }}
       fail-fast: false
     defaults:
       run:
@@ -243,6 +245,9 @@ jobs:
             --variable cache_path=/home/rfay/cache/drupal-core-seed \
             --variable github_token=${{ secrets.GITHUB_TOKEN }}
 
+      - name: Reap predecessor workspaces (cancelled prior runs of this cell)
+        run: ./scripts/ci-reap-family.sh "gd-${{ matrix.drupal_version }}-" "${{ env.WORKSPACE_NAME }}"
+
       - name: Create workspace
         run: |
           coder create ${{ env.WORKSPACE_NAME }} \
@@ -495,6 +500,9 @@ jobs:
             --variable cache_path=/home/rfay/cache/drupal-core-seed \
             --variable github_token=${{ secrets.GITHUB_TOKEN }}
 
+      - name: Reap predecessor workspaces (cancelled prior runs of this cell)
+        run: ./scripts/ci-reap-family.sh "gd-fork-" "${{ env.WORKSPACE_NAME }}"
+
       - name: Create workspace
         run: |
           coder create ${{ env.WORKSPACE_NAME }} \

.github/workflows/integration-test.yml

@@ -352,6 +352,9 @@ jobs:
             --variable workspace_image_registry=index.docker.io/ddev/coder-ddev \
             ${{ matrix.extra_vars }}
 
+      - name: Reap predecessor workspaces (cancelled prior runs of this cell)
+        run: ./scripts/ci-reap-family.sh "gh-${{ matrix.ws_name }}-" "${{ env.WORKSPACE_NAME }}"
+
       - name: Create workspace
         if: ${{ matrix.template != 'freeform' }}
         run: |

image/Dockerfile

@@ -175,7 +175,8 @@ RUN /home/linuxbrew/.linuxbrew/bin/brew install \
   claude-code \
   go \
   golangci-lint \
-  yq
+  yq \
+  zellij
 
 USER root
 # Symlink brew-installed tools to /usr/local/bin so they're available system-wide

scripts/ci-reap-family.sh

@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+# Delete predecessor CI workspaces in the same name "family" before a test job
+# creates its own workspace.
+#
+# Why this exists:
+#   cancel-in-progress cancels the GitHub run on a new push to a PR, but the
+#   workspace that the cancelled run created keeps running on staging (the
+#   cancelled job's `coder delete` step never reaches the server). The
+#   replacement run is the natural place to clean up its predecessor: it knows
+#   its own family and can delete any older sibling before creating its own.
+#
+#   Workspace names are <family>-<run_number>-<run_attempt> (e.g.
+#   gc-pathauto-d11-152-1). The family is the stable part shared across runs of
+#   the same matrix cell. Passing the family prefix here deletes every ci-bot
+#   workspace that starts with it EXCEPT the one this run is about to create.
+#
+# Usage:
+#   ci-reap-family.sh <family-prefix> [keep-name]
+#     <family-prefix>  e.g. "gc-pathauto-d11-"  (trailing dash recommended)
+#     [keep-name]      workspace name to NOT delete (this run's own); optional
+#
+# Environment:
+#   CI_OWNER   Coder username that owns CI workspaces (default: ci-bot)
+#
+# Always exits 0 — predecessor cleanup is best-effort and must never fail a run.
+
+set -uo pipefail
+
+FAMILY="${1:?usage: ci-reap-family.sh <family-prefix> [keep-name]}"
+KEEP="${2:-}"
+CI_OWNER="${CI_OWNER:-ci-bot}"
+
+if ! workspaces_json=$(coder list --all --output json 2>/dev/null); then
+  echo "WARN: 'coder list' failed; skipping predecessor reap" >&2
+  exit 0
+fi
+
+mapfile -t victims < <(echo "$workspaces_json" |
+  jq -r --arg owner "$CI_OWNER" --arg fam "$FAMILY" --arg keep "$KEEP" \
+    '.[] | select(.owner_name==$owner) | .name
+       | select(startswith($fam)) | select(. != $keep)')
+
+if [[ ${#victims[@]} -eq 0 ]]; then
+  echo "No predecessor workspaces for family '$FAMILY'"
+  exit 0
+fi
+
+for name in "${victims[@]}"; do
+  echo "Reaping predecessor: $name"
+  coder delete "$name" --yes || echo "  WARN: delete failed for $name" >&2
+done
+
+exit 0

scripts/ci-reap-staging.sh

@@ -0,0 +1,130 @@
+#!/usr/bin/env bash
+# Reap stale CI workspaces and accumulated template versions on the staging
+# Coder instance.
+#
+# Why this exists:
+#   Each integration-test run pushes a template version and creates a workspace,
+#   both owned by the CI user (ci-bot). The per-run cleanup (coder delete +
+#   coder templates versions archive) lives in `if: always()` steps inside the
+#   test job. When a run is cancelled (cancel-in-progress on a new push), force
+#   killed, or its workspace lands in `Failed`, those steps never reach the
+#   server — GitHub frees its runner but staging keeps the running workspace and
+#   the un-archived version. Over many PR pushes this backs staging up.
+#
+#   This janitor is decoupled from any individual test job: it runs on a schedule
+#   and reaps by owner + state + age, so cancellation can never leave an orphan
+#   alive longer than one janitor interval. It also handles merged / closed /
+#   abandoned PRs, where no replacement run ever comes to clean up.
+#
+# What it reaps (only workspaces owned by $CI_OWNER, default ci-bot):
+#   - failed / stopped / canceled  -> deleted regardless of age
+#   - running                      -> deleted if its latest build is older than
+#                                     $AGE_MINUTES (default 20)
+#   - pending / starting / stopping / canceling / deleting -> left alone
+#                                     (transitional; caught on a later pass)
+# Then archives all unused template versions for each CI template.
+#
+# By default runs in dry-run mode (prints what would be deleted).
+# Pass --force (or set DRY_RUN=false) to actually delete/archive.
+#
+# Requires: coder CLI authenticated as a template-admin who owns the CI workspaces.
+#
+# Environment overrides:
+#   CI_OWNER      Coder username that owns CI workspaces (default: ci-bot)
+#   AGE_MINUTES   Age threshold in minutes for running workspaces (default: 20)
+#   DRY_RUN       true|false (default: true; --force sets false)
+#   TEMPLATES     space-separated template names to archive versions for
+#
+# Usage:
+#   ./scripts/ci-reap-staging.sh                 # dry run
+#   ./scripts/ci-reap-staging.sh --force         # actually reap
+#   AGE_MINUTES=60 ./scripts/ci-reap-staging.sh --force
+
+set -euo pipefail
+
+CI_OWNER="${CI_OWNER:-ci-bot}"
+AGE_MINUTES="${AGE_MINUTES:-20}"
+DRY_RUN="${DRY_RUN:-true}"
+TEMPLATES="${TEMPLATES:-drupal-core drupal-contrib freeform user-defined-web}"
+
+for arg in "$@"; do
+  case "$arg" in
+    --force) DRY_RUN=false ;;
+    --help | -h)
+      sed -n '2,40p' "$0" | sed 's/^# \?//'
+      exit 0
+      ;;
+    *)
+      echo "Unknown argument: $arg" >&2
+      exit 1
+      ;;
+  esac
+done
+
+if [[ "$DRY_RUN" == "true" ]]; then
+  echo "DRY RUN — pass --force (or DRY_RUN=false) to actually reap"
+  echo
+fi
+
+now=$(date +%s)
+cutoff=$((now - AGE_MINUTES * 60))
+
+if ! workspaces_json=$(coder list --all --output json 2>/dev/null); then
+  echo "ERROR: 'coder list --all --output json' failed. Is the coder CLI authenticated?" >&2
+  exit 1
+fi
+
+# --- Workspaces ---
+# Emit: name <TAB> status <TAB> build_created  for $CI_OWNER-owned workspaces only.
+reaped=0 kept=0
+while IFS=$'\t' read -r name status build_created; do
+  [[ -z "$name" ]] && continue
+
+  reap_reason=""
+  case "$status" in
+    failed | stopped | canceled)
+      reap_reason="status=$status"
+      ;;
+    running | started)
+      build_epoch=$(date -d "$build_created" +%s 2>/dev/null || echo 0)
+      if [[ "$build_epoch" -gt 0 && "$build_epoch" -lt "$cutoff" ]]; then
+        age_min=$(((now - build_epoch) / 60))
+        reap_reason="running ${age_min}m (>${AGE_MINUTES}m)"
+      fi
+      ;;
+    *)
+      # pending / starting / stopping / canceling / deleting — transitional, skip
+      ;;
+  esac
+
+  if [[ -n "$reap_reason" ]]; then
+    echo "REAP  $name  [$reap_reason]"
+    reaped=$((reaped + 1))
+    if [[ "$DRY_RUN" != "true" ]]; then
+      coder delete "$name" --yes || echo "  WARN: delete failed for $name" >&2
+    fi
+  else
+    kept=$((kept + 1))
+  fi
+done < <(echo "$workspaces_json" |
+  jq -r --arg owner "$CI_OWNER" \
+    '.[] | select(.owner_name==$owner) | [.name, .latest_build.status, .latest_build.created_at] | @tsv')
+
+echo
+echo "Workspaces: reaped=$reaped kept=$kept (owner=$CI_OWNER, age=${AGE_MINUTES}m)"
+echo
+
+# --- Template versions ---
+# `coder templates archive <t> --all` archives every unused version (never the
+# active one, never one with a live workspace), which is exactly what we want.
+echo "Archiving unused template versions..."
+for t in $TEMPLATES; do
+  if [[ "$DRY_RUN" == "true" ]]; then
+    echo "  would: coder templates archive $t --all --yes"
+  else
+    coder templates archive "$t" --all --yes || echo "  WARN: archive failed for $t" >&2
+  fi
+done
+
+echo
+echo "Done."