diff --git a/.github/workflows/workspace-lifecycle-cleanup.yml b/.github/workflows/workspace-lifecycle-cleanup.yml new file mode 100644 index 0000000..ab39be3 --- /dev/null +++ b/.github/workflows/workspace-lifecycle-cleanup.yml @@ -0,0 +1,103 @@ +name: Workspace lifecycle cleanup + +# Daily janitor for coder.ddev.com (production). Notifies owners of idle +# workspaces, then deletes them if they stay idle after the notice period. +# See scripts/workspace-lifecycle-cleanup.sh for the full policy. +# +# Deleting a workspace via `coder delete` also frees its dind-cache Docker +# volume on the host (the destroy provisioner in each template removes it), +# which is what actually reclaims disk space on /data. +# +# Requires: +# Repository variable: PROD_CODER_URL - https://coder.ddev.com +# Repository secret: OP_SERVICE_ACCOUNT_TOKEN - 1Password service account +# 1Password items (adjust vault/item names to match your 1Password setup): +# op://coder-ddev-secrets/PROD_CODER_SESSION_TOKEN/credential +# op://coder-ddev-secrets/MAILGUN_API_KEY/credential +# op://coder-ddev-secrets/MAILGUN_DOMAIN/credential +# +# PROD_CODER_SESSION_TOKEN must belong to a user with the `owner` role — +# this deployment has no Premium license, so `owner` is the only built-in +# role that can list every user's workspaces and delete workspaces it +# doesn't own. Provision a dedicated service account for this rather than +# reusing a personal token; see "Provisioning the PROD_CODER_SESSION_TOKEN +# credential" in docs/admin/operations-guide.md for the exact commands. +# +# The state file at scripts/state/workspace-lifecycle-state.json is +# committed back to the branch after every --force run so the notify/delete +# grace period survives across ephemeral runner instances. + +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + +concurrency: + group: workspace-lifecycle-cleanup + cancel-in-progress: false + +permissions: + contents: write + +on: + schedule: + - cron: '17 6 * * *' + workflow_dispatch: + inputs: + dry_run: + description: 'Dry run (list only, notify/delete nothing)' + type: boolean + required: false + default: true + notify_days: + description: 'Idle days before the first notice' + type: string + required: false + default: '7' + delete_after_days: + description: 'Days after notice before deletion' + type: string + required: false + default: '7' + +jobs: + cleanup: + name: Notify and reap idle workspaces + runs-on: ubuntu-latest + defaults: + run: + shell: bash -euo pipefail {0} + steps: + - uses: actions/checkout@v6 + + - name: Load 1Password secrets + uses: 1password/load-secrets-action@v4 + with: + export-env: true + env: + OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} + PROD_CODER_SESSION_TOKEN: 'op://coder-ddev-secrets/PROD_CODER_SESSION_TOKEN/credential' + MAILGUN_API_KEY: 'op://coder-ddev-secrets/MAILGUN_API_KEY/credential' + MAILGUN_DOMAIN: 'op://coder-ddev-secrets/MAILGUN_DOMAIN/credential' + + - uses: coder/setup-action@v1 + with: + access_url: ${{ vars.PROD_CODER_URL }} + coder_session_token: ${{ env.PROD_CODER_SESSION_TOKEN }} + + - name: Notify and reap idle workspaces + env: + DRY_RUN: ${{ github.event.inputs.dry_run || 'false' }} + NOTIFY_DAYS: ${{ github.event.inputs.notify_days || '7' }} + DELETE_AFTER_DAYS: ${{ github.event.inputs.delete_after_days || '7' }} + run: ./scripts/workspace-lifecycle-cleanup.sh $([[ "$DRY_RUN" == "true" ]] || echo --force) + + - name: Commit updated state file + run: | + if ! git diff --quiet -- scripts/state/workspace-lifecycle-state.json; then + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git add scripts/state/workspace-lifecycle-state.json + git commit -m "chore(workspace-cleanup): update lifecycle state [skip ci]" + git push + else + echo "No state changes to commit." + fi diff --git a/CLAUDE.md b/CLAUDE.md index cf313dc..8833a21 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -358,6 +358,7 @@ Additional logs in workspace: - `image/scripts/.ddev/global_config.yaml` - DDEV defaults copied into workspaces - `scripts/coder-delete-workspace-dir.sh` - Sudo wrapper for workspace host dir cleanup (must be installed on server) - `scripts/cleanup-deleted-workspaces.sh` - Manual cleanup for orphaned workspace dirs/volumes +- `scripts/workspace-lifecycle-cleanup.sh` - Notifies then deletes idle workspaces on coder.ddev.com (see `.github/workflows/workspace-lifecycle-cleanup.yml`) - `VERSION` - Image version used by all templates (read automatically by Makefile) - `openspec/project.md` - Project conventions and constraints - `openspec/AGENTS.md` - OpenSpec workflow instructions diff --git a/docs/admin/operations-guide.md b/docs/admin/operations-guide.md index 1b59241..7c88d90 100644 --- a/docs/admin/operations-guide.md +++ b/docs/admin/operations-guide.md @@ -417,6 +417,56 @@ docker system df --- +### Automated Idle Workspace Cleanup + +A scheduled GitHub Actions workflow (`.github/workflows/workspace-lifecycle-cleanup.yml`) runs `scripts/workspace-lifecycle-cleanup.sh` daily against coder.ddev.com to keep idle workspaces from accumulating and slowly filling `/data` with `*-dind-cache` Docker volumes (each workspace keeps its full Docker-in-Docker cache volume until it's deleted, even while stopped). + +Policy per workspace, based on `last_used_at`: + +- **7 days idle** — owner gets a notice email (via Mailgun) explaining the workspace will be deleted, and pointing to the auth/access announcement if they can no longer log in. +- **7 more days idle after the notice** — the workspace is deleted with `coder delete --yes`. +- If the owner uses the workspace again before deletion, the pending notice is cleared automatically. + +State (which workspaces have been notified and when) is tracked in `scripts/state/workspace-lifecycle-state.json`, which the workflow commits back to the repo after every run so the grace period survives across runs. + +```bash +# Dry run — shows what would be notified/deleted, sends no email, deletes nothing +./scripts/workspace-lifecycle-cleanup.sh + +# Actually send notices, delete workspaces past their grace period, and persist state +./scripts/workspace-lifecycle-cleanup.sh --force +``` + +**Prerequisites:** `coder` CLI authenticated against coder.ddev.com as a user with the `owner` role (see below); `MAILGUN_API_KEY` and `MAILGUN_DOMAIN` set for `--force` runs. See the script header for all environment overrides (`NOTIFY_DAYS`, `DELETE_AFTER_DAYS`, `EXCLUDE_OWNERS`, etc.). + +The workflow can also be triggered manually (`workflow_dispatch`) with a `dry_run` input for testing. + +#### Provisioning the `PROD_CODER_SESSION_TOKEN` credential + +The script needs to list *every* user's workspaces (`coder list --all`) and delete workspaces it doesn't own. On this deployment (no Premium license, so no custom RBAC roles), `owner` is the only built-in role that can do both — there's no narrower "workspace admin" role available. That makes this token effectively full site-admin on production, so it's provisioned as a dedicated non-human account rather than a personal token: + +```bash +# 1. Create a machine identity — no GitHub OAuth login required +coder users create --username workspace-janitor \ + --email workspace-janitor@ddev.com \ + --full-name "Workspace Lifecycle Janitor" \ + --login-type none + +# 2. Grant it owner — the only built-in role that covers list-all + delete-any-workspace +coder users edit-roles workspace-janitor --roles owner --yes + +# 3. Mint a long-lived token for it (run as an existing owner/admin, e.g. your own account) +coder tokens create -u workspace-janitor --name workspace-lifecycle-ci --lifetime 8760h +``` + +Store the resulting token as the `PROD_CODER_SESSION_TOKEN` secret referenced by the workflow. Using a dedicated account rather than a personal token keeps deletions attributable to the bot (not an individual) in the audit log, and means the janitor doesn't break if the admin's own account is later deactivated or re-authenticated. + +Check the server's configured max token lifetime before choosing `--lifetime` — if it's capped below a year, use the max allowed and set a reminder to rotate the token before it expires, since an expired token fails the workflow silently until someone notices. + +`--login-type none` is deprecated in favor of `--service-account` (Premium-only) but remains functional for this purpose. + +--- + ### Orphaned Workspace Cleanup When a workspace is deleted, the destroy provisioner automatically removes the host directory at `/coder-workspaces/-`. Directories can still be orphaned if the provisioner fails or for workspaces deleted before the provisioner was added. Run the cleanup script to reclaim disk space in those cases. diff --git a/scripts/workspace-lifecycle-cleanup.sh b/scripts/workspace-lifecycle-cleanup.sh new file mode 100755 index 0000000..c93874e --- /dev/null +++ b/scripts/workspace-lifecycle-cleanup.sh @@ -0,0 +1,284 @@ +#!/usr/bin/env bash +# Notify and delete idle workspaces on coder.ddev.com. +# +# Why this exists: +# Stopping a workspace does not free its Docker volume — each Sysbox +# workspace keeps a multi-GB dind-cache volume until the workspace is +# deleted (the destroy provisioner in each template removes it). With no +# lifecycle policy, stopped workspaces accumulate forever and slowly fill +# /data on the host. This janitor enforces: notify at N days idle, delete +# 7 days after the notice if the workspace is still untouched. +# +# Policy (state machine per workspace, keyed by workspace id): +# - Not yet notified, age(last_used_at) >= NOTIFY_DAYS +# -> email the owner, record notified_at=now in the state file +# - Already notified, but last_used_at advanced past the notice +# -> the owner came back; clear the state entry (no deletion) +# - Already notified, DELETE_AFTER_DAYS have passed since notified_at, +# and last_used_at still hasn't advanced +# -> `coder delete`, drop the state entry +# - Already notified, still within the grace window +# -> leave alone +# - State entry exists for a workspace that no longer exists +# -> drop the stale entry +# +# The very first run against a fleet with no history only sends notices — +# nothing is old enough to be "already notified" yet, so nothing gets +# deleted until a full DELETE_AFTER_DAYS grace period has elapsed from the +# first notice. That is the intended one-time "grandfather" behavior; no +# separate rollout mode is needed. +# +# By default runs in dry-run mode (prints what would happen, does not send +# email, delete workspaces, or write the state file). Pass --force to act. +# +# Requires: coder CLI (authenticated), jq, curl, a Mailgun account. +# +# Environment: +# CODER_URL Coder deployment to operate on (informational; auth +# is whatever the coder CLI is currently logged into) +# NOTIFY_DAYS Idle days before the first notice (default: 7) +# DELETE_AFTER_DAYS Days after notice before deletion (default: 7) +# EXCLUDE_OWNERS Space-separated owner usernames to skip (default: ci-bot) +# STATE_FILE Path to the JSON state file +# (default: scripts/state/workspace-lifecycle-state.json) +# MAILGUN_API_KEY Mailgun private API key (required to send, not for dry-run) +# MAILGUN_DOMAIN Mailgun sending domain, e.g. mg.ddev.com +# MAILGUN_BASE_URL Mailgun API base (default: https://api.mailgun.net/v3; +# use https://api.eu.mailgun.net/v3 for EU-region domains) +# MAILGUN_FROM From header (default: "DDEV Coder ") +# ANNOUNCE_URL Blog post explaining the auth change +# (default: https://ddev.com/blog/coder-ddev-com-announcement/) +# +# Usage: +# ./scripts/workspace-lifecycle-cleanup.sh # dry run +# ./scripts/workspace-lifecycle-cleanup.sh --force # actually notify/delete + +set -euo pipefail + +NOTIFY_DAYS="${NOTIFY_DAYS:-7}" +DELETE_AFTER_DAYS="${DELETE_AFTER_DAYS:-7}" +EXCLUDE_OWNERS="${EXCLUDE_OWNERS:-ci-bot}" +STATE_FILE="${STATE_FILE:-scripts/state/workspace-lifecycle-state.json}" +MAILGUN_BASE_URL="${MAILGUN_BASE_URL:-https://api.mailgun.net/v3}" +MAILGUN_FROM="${MAILGUN_FROM:-DDEV Coder }" +ANNOUNCE_URL="${ANNOUNCE_URL:-https://ddev.com/blog/coder-ddev-com-announcement/}" +FORCE=false + +for arg in "$@"; do + case "$arg" in + --force) FORCE=true ;; + --help | -h) + sed -n '2,45p' "$0" | sed 's/^# \?//' + exit 0 + ;; + *) + echo "Unknown argument: $arg" >&2 + exit 1 + ;; + esac +done + +if [[ "$FORCE" == false ]]; then + echo "DRY RUN — pass --force to actually notify/delete and persist state" + echo +fi + +if [[ "$FORCE" == true ]]; then + : "${MAILGUN_API_KEY:?MAILGUN_API_KEY is required with --force}" + : "${MAILGUN_DOMAIN:?MAILGUN_DOMAIN is required with --force}" +fi + +# Portable date handling: prefer GNU date (Linux, or `gdate` from +# coreutils on macOS); fall back to BSD date's `-j -f` / `-r` syntax. +if date -d "1970-01-01T00:00:00Z" +%s >/dev/null 2>&1; then + DATE_BIN=(date) +elif command -v gdate >/dev/null 2>&1; then + DATE_BIN=(gdate) +else + DATE_BIN=() +fi + +iso_to_epoch() { + local iso="$1" + if [[ ${#DATE_BIN[@]} -gt 0 ]]; then + "${DATE_BIN[@]}" -d "$iso" +%s 2>/dev/null && return + fi + date -j -f "%Y-%m-%dT%H:%M:%S" "${iso%%.*}" +%s 2>/dev/null || echo 0 +} + +epoch_to_ymd() { + local epoch="$1" + if [[ ${#DATE_BIN[@]} -gt 0 ]]; then + "${DATE_BIN[@]}" -d "@${epoch}" +%Y-%m-%d 2>/dev/null && return + fi + date -j -r "$epoch" +%Y-%m-%d 2>/dev/null || echo "unknown" +} + +epoch_to_iso() { + local epoch="$1" + if [[ ${#DATE_BIN[@]} -gt 0 ]]; then + "${DATE_BIN[@]}" -u -d "@${epoch}" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null && return + fi + date -u -j -r "$epoch" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null +} + +mkdir -p "$(dirname "$STATE_FILE")" +if [[ -f "$STATE_FILE" ]]; then + state_json=$(cat "$STATE_FILE") +else + state_json='{}' +fi + +if ! workspaces_json=$(coder list --all --output json 2>/dev/null); then + echo "ERROR: 'coder list --all --output json' failed. Is the coder CLI authenticated?" >&2 + exit 1 +fi + +workspace_count=$(echo "$workspaces_json" | jq 'length') +if [[ "$workspace_count" -eq 0 ]]; then + echo "ERROR: 'coder list' returned zero workspaces. Refusing to proceed — this looks like an auth or connectivity problem, not an empty fleet." >&2 + exit 1 +fi + +if ! users_json=$(coder users list --output json 2>/dev/null); then + echo "ERROR: 'coder users list --output json' failed." >&2 + exit 1 +fi + +now=$(date +%s) + +send_notice_email() { + local to_email="$1" owner_name="$2" workspace_name="$3" last_used_human="$4" delete_date_human="$5" + + local subject="Action needed: your coder.ddev.com workspace \"${workspace_name}\" will be deleted on ${delete_date_human}" + local body + body=$( + cat </dev/null + fi +} + +notified=0 revived=0 deleted=0 pending=0 kept=0 pruned=0 + +# --- Prune state entries for workspaces that no longer exist --- +current_ids=$(echo "$workspaces_json" | jq -r '.[].id') +new_state_json="$state_json" +while IFS= read -r stale_id; do + [[ -z "$stale_id" ]] && continue + if ! grep -qx "$stale_id" <<<"$current_ids"; then + echo "PRUNE state entry for deleted workspace $stale_id" + new_state_json=$(echo "$new_state_json" | jq --arg id "$stale_id" 'del(.[$id])') + pruned=$((pruned + 1)) + fi +done < <(echo "$state_json" | jq -r 'keys[]') +state_json="$new_state_json" + +# --- Walk workspaces --- +while IFS=$'\t' read -r id name owner last_used_at; do + [[ -z "$id" ]] && continue + + if grep -qx "$owner" <<<"$(tr ' ' '\n' <<<"$EXCLUDE_OWNERS")"; then + continue + fi + + last_epoch=$(iso_to_epoch "$last_used_at") + age_days=$(((now - last_epoch) / 86400)) + + entry=$(echo "$state_json" | jq --arg id "$id" '.[$id]') + + if [[ "$entry" != "null" ]]; then + notified_at=$(echo "$entry" | jq -r '.notified_at') + notified_epoch=$(iso_to_epoch "$notified_at") + + if [[ "$age_days" -lt "$NOTIFY_DAYS" ]]; then + echo "REVIVE $name (owner=$owner) — used again since notice, clearing pending deletion" + revived=$((revived + 1)) + state_json=$(echo "$state_json" | jq --arg id "$id" 'del(.[$id])') + continue + fi + + since_notice_days=$(((now - notified_epoch) / 86400)) + if [[ "$since_notice_days" -ge "$DELETE_AFTER_DAYS" ]]; then + echo "DELETE $name (owner=$owner) — idle ${age_days}d, notified ${since_notice_days}d ago" + deleted=$((deleted + 1)) + if [[ "$FORCE" == true ]]; then + coder delete "$name" --yes || echo " WARN: delete failed for $name" >&2 + fi + state_json=$(echo "$state_json" | jq --arg id "$id" 'del(.[$id])') + else + echo "PENDING $name (owner=$owner) — notified ${since_notice_days}d ago, deletes in $((DELETE_AFTER_DAYS - since_notice_days))d unless used" + pending=$((pending + 1)) + fi + continue + fi + + if [[ "$age_days" -ge "$NOTIFY_DAYS" ]]; then + owner_email=$(echo "$users_json" | jq -r --arg u "$owner" '.[] | select(.username==$u) | .email') + owner_display=$(echo "$users_json" | jq -r --arg u "$owner" '.[] | select(.username==$u) | .name') + if [[ -z "$owner_email" ]]; then + echo " WARN: no email found for owner $owner (workspace $name), skipping notice" >&2 + continue + fi + + last_used_human=$(epoch_to_ymd "$last_epoch") + delete_epoch=$((now + DELETE_AFTER_DAYS * 86400)) + delete_date_human=$(epoch_to_ymd "$delete_epoch") + + echo "NOTIFY $name (owner=$owner <$owner_email>) — idle ${age_days}d, last used $last_used_human, deletes $delete_date_human unless used" + notified=$((notified + 1)) + send_notice_email "$owner_email" "${owner_display:-$owner}" "$name" "$last_used_human" "$delete_date_human" + + state_json=$(echo "$state_json" | jq --arg id "$id" --arg at "$(epoch_to_iso "$now")" --arg name "$name" --arg owner "$owner" \ + '.[$id] = {notified_at: $at, name: $name, owner: $owner}') + else + kept=$((kept + 1)) + fi +done < <(echo "$workspaces_json" | jq -r '.[] | [.id, .name, .owner_name, .last_used_at] | @tsv') + +echo +echo "Summary: notified=$notified pending=$pending deleted=$deleted revived=$revived kept=$kept pruned=$pruned" + +if [[ "$FORCE" == true ]]; then + echo "$state_json" | jq '.' >"$STATE_FILE" + echo "State written to $STATE_FILE" +else + echo + echo "Re-run with --force to send notices, delete workspaces, and persist state." +fi