From 9cf09a90788202fb2e944ee237288996e02cd585 Mon Sep 17 00:00:00 2001 From: Keith Kraus Date: Fri, 31 Jul 2026 01:22:19 -0400 Subject: [PATCH 1/4] ci: activate dependency-aware package builds and tests --- .github/workflows/ci.yml | 436 ++++++++++++++++++++++++++++++------ ci/tools/compute_ci_plan.py | 147 ++++++++++++ 2 files changed, 515 insertions(+), 68 deletions(-) create mode 100644 ci/tools/compute_ci_plan.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2aadf222306..0be7f9445b5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -72,24 +72,31 @@ jobs: echo "skip=${skip}" >> "$GITHUB_OUTPUT" echo "doc_only=${doc_only}" >> "$GITHUB_OUTPUT" - # Detect which top-level modules were touched by the PR so downstream build - # and test jobs can avoid rebuilding/retesting modules unaffected by the - # change. See issue #299. + # Detect which packages were touched by the PR so downstream build and test + # jobs can avoid rebuilding/retesting packages unaffected by the change. + # See issue #299. # # Dependency graph (verified in pyproject.toml files): # cuda_pathfinder -> (no internal deps) # cuda_bindings -> cuda_pathfinder # cuda_core -> cuda_pathfinder, cuda_bindings - # cuda_python -> cuda_bindings (meta package) + # cuda_python -> cuda_pathfinder, cuda_bindings, cuda_core (meta package) # # A change to cuda_pathfinder (or shared infra) forces a rebuild of every # downstream module. A change to cuda_bindings forces rebuild of cuda_core. - # A change to cuda_core alone skips rebuilding/retesting cuda_bindings. + # A change to cuda_core alone skips rebuilding/retesting cuda_bindings and + # cuda_pathfinder, but still retests the downstream cuda-python metapackage. + # CI/planner changes are shared by design, so this implementation runs the + # full pipeline; exercise selective cases in follow-up package-only PRs. # On push to main, tag refs, schedule, or workflow_dispatch events we # unconditionally run everything because there is no meaningful "changed # paths" baseline for those events. detect-changes: runs-on: ubuntu-latest + permissions: + actions: read + contents: read + pull-requests: read outputs: bindings: ${{ steps.compose.outputs.bindings }} core: ${{ steps.compose.outputs.core }} @@ -100,10 +107,14 @@ jobs: build_bindings: ${{ steps.compose.outputs.build_bindings }} build_core: ${{ steps.compose.outputs.build_core }} build_pathfinder: ${{ steps.compose.outputs.build_pathfinder }} + build_python: ${{ steps.compose.outputs.build_python }} test_bindings: ${{ steps.compose.outputs.test_bindings }} test_core: ${{ steps.compose.outputs.test_core }} test_pathfinder: ${{ steps.compose.outputs.test_pathfinder }} pr_merge_base: ${{ steps.filter.outputs.merge_base }} + test_python: ${{ steps.compose.outputs.test_python }} + baseline_run_id: ${{ steps.compose.outputs.baseline_run_id }} + baseline_sha: ${{ steps.compose.outputs.baseline_sha }} steps: - name: Checkout repository uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 @@ -136,52 +147,177 @@ jobs: # off by `if:`, so `BASE_REF` is never consumed there. BASE_REF: ${{ steps.pr-info.outputs.pr-info && fromJSON(steps.pr-info.outputs.pr-info).base.ref || '' }} run: | - # Diff against the merge base with the PR's actual target branch. - # Uses merge-base so diverged branches only show files changed on - # the PR side, not upstream commits. + set -euo pipefail if [[ -z "${BASE_REF}" ]]; then echo "Could not resolve PR base branch from get-pr-info output" >&2 exit 1 fi + + # Diff against the merge base with the PR's actual target branch. + # Disabling rename detection reports both sides of a cross-package + # move, which prevents the source package from being skipped. base=$(git merge-base HEAD "origin/${BASE_REF}") - changed=$(git diff --name-only "$base"...HEAD) + git diff --no-renames --name-only -z "$base"...HEAD > changed-paths + python ci/tools/compute_ci_plan.py changed-paths >> "$GITHUB_OUTPUT" + echo "merge_base=${base}" >> "$GITHUB_OUTPUT" + + { + echo "### Selective CI changed paths" + echo + tr '\0' '\n' < changed-paths | sed 's/^/- `/' | sed 's/$/`/' + } >> "$GITHUB_STEP_SUMMARY" + + - name: Resolve reusable base artifacts + id: baseline + if: ${{ startsWith(github.ref_name, 'pull-request/') }} + env: + BASE_REF: ${{ steps.pr-info.outputs.pr-info && fromJSON(steps.pr-info.outputs.pr-info).base.ref || '' }} + GH_TOKEN: ${{ github.token }} + run: | + set -uo pipefail + + unavailable() { + echo "available=false" >> "$GITHUB_OUTPUT" + echo "No complete reusable artifact set was found; this run will build and test everything." >> "$GITHUB_STEP_SUMMARY" + exit 0 + } + + if [[ -z "${BASE_REF}" ]]; then + unavailable + fi + + merge_base=$(git merge-base HEAD "origin/${BASE_REF}") + if ! runs=$(gh run list \ + --repo "${{ github.repository }}" \ + --branch "${BASE_REF}" \ + --commit "${merge_base}" \ + --event push \ + --workflow ci.yml \ + --status success \ + --limit 1 \ + --json databaseId,headSha,createdAt); then + unavailable + fi + + # Reuse only artifacts produced from the exact commit used as the + # PR diff base. Using the latest base-branch run is unsafe for a PR + # that was opened before newer changes landed on that branch. + run_id=$(jq -r '.[0].databaseId // empty' <<< "$runs") + baseline_sha=$(jq -r '.[0].headSha // empty' <<< "$runs") + if [[ -z "${run_id}" || "${baseline_sha}" != "${merge_base}" ]]; then + unavailable + fi + + if ! artifact_names=$(gh api \ + "repos/${{ github.repository }}/actions/runs/${run_id}/artifacts?per_page=100" \ + --paginate \ + --jq '.artifacts[] | select(.expired == false) | .name'); then + unavailable + fi - has_match() { - grep -qE "$1" <<< "$changed" && echo true || echo false + has_artifact() { + grep -Fxq "$1" <<< "$artifact_names" } + missing=() + for name in cuda-pathfinder-wheel cuda-python-wheel; do + has_artifact "$name" || missing+=("$name") + done + + cuda_version=$(yq '.cuda.build.version' ci/versions.yml) + if ! python_versions=$(yq -r '.jobs.build.strategy.matrix."python-version"[]' .github/workflows/build-wheel.yml); then + unavailable + fi + if [[ -z "${python_versions}" ]]; then + unavailable + fi + while IFS= read -r python_version; do + python=${python_version//./} + for platform in linux-64 linux-aarch64 win-64; do + binding="cuda-bindings-python${python}-cuda${cuda_version}-${platform}-${baseline_sha}" + core="cuda-core-python${python}-${platform}-${baseline_sha}" + has_artifact "$binding" || missing+=("$binding") + has_artifact "$core" || missing+=("$core") + done + done <<< "${python_versions}" + + if (( ${#missing[@]} != 0 )); then + printf 'Missing reusable artifact: %s\n' "${missing[@]}" >&2 + unavailable + fi + { - echo "bindings=$(has_match '^cuda_bindings/')" - echo "core=$(has_match '^cuda_core/')" - echo "pathfinder=$(has_match '^cuda_pathfinder/')" - echo "python_meta=$(has_match '^cuda_python/')" - echo "test_helpers=$(has_match '^cuda_python_test_helpers/')" - echo "shared=$(has_match '^(\.github/|ci/|scripts/|toolshed/|conftest\.py$|pyproject\.toml$|pixi\.(toml|lock)$|pytest\.ini$|ruff\.toml$)')" - echo "merge_base=${base}" + echo "available=true" + echo "run_id=${run_id}" + echo "sha=${baseline_sha}" } >> "$GITHUB_OUTPUT" + { + echo + echo "Reusable artifacts: run \`${run_id}\` at \`${baseline_sha}\` on \`${BASE_REF}\`." + } >> "$GITHUB_STEP_SUMMARY" - name: Compose gating outputs id: compose env: IS_PR: ${{ startsWith(github.ref_name, 'pull-request/') }} - BINDINGS: ${{ steps.filter.outputs.bindings || 'false' }} - CORE: ${{ steps.filter.outputs.core || 'false' }} - PATHFINDER: ${{ steps.filter.outputs.pathfinder || 'false' }} - PYTHON_META: ${{ steps.filter.outputs.python_meta || 'false' }} - TEST_HELPERS: ${{ steps.filter.outputs.test_helpers || 'false' }} - SHARED: ${{ steps.filter.outputs.shared || 'false' }} + BASELINE_AVAILABLE: ${{ steps.baseline.outputs.available || 'false' }} + BASELINE_RUN_ID: ${{ steps.baseline.outputs.run_id }} + BASELINE_SHA: ${{ steps.baseline.outputs.sha }} + BINDINGS: ${{ steps.filter.outputs.bindings_source }} + CORE: ${{ steps.filter.outputs.core_source }} + PATHFINDER: ${{ steps.filter.outputs.pathfinder_source }} + PYTHON_META: ${{ steps.filter.outputs.python_source }} + TEST_HELPERS: ${{ steps.filter.outputs.test_helpers }} + SHARED: ${{ steps.filter.outputs.shared }} + BUILD_BINDINGS: ${{ steps.filter.outputs.build_bindings }} + BUILD_CORE: ${{ steps.filter.outputs.build_core }} + BUILD_PATHFINDER: ${{ steps.filter.outputs.build_pathfinder }} + BUILD_PYTHON: ${{ steps.filter.outputs.build_python }} + TEST_BINDINGS: ${{ steps.filter.outputs.test_bindings }} + TEST_CORE: ${{ steps.filter.outputs.test_core }} + TEST_PATHFINDER: ${{ steps.filter.outputs.test_pathfinder }} + TEST_PYTHON: ${{ steps.filter.outputs.test_python }} run: | set -euxo pipefail - # Non-PR events (push to main, tag push, schedule, workflow_dispatch) - # always exercise the full pipeline because there is no baseline for - # a meaningful diff. - if [[ "${IS_PR}" != "true" ]]; then + planner_valid=true + if [[ "${IS_PR}" == "true" ]]; then + for value in \ + "${BINDINGS}" "${CORE}" "${PATHFINDER}" "${PYTHON_META}" \ + "${TEST_HELPERS}" "${SHARED}" \ + "${BUILD_BINDINGS}" "${BUILD_CORE}" "${BUILD_PATHFINDER}" "${BUILD_PYTHON}" \ + "${TEST_BINDINGS}" "${TEST_CORE}" "${TEST_PATHFINDER}" "${TEST_PYTHON}"; do + if [[ "${value}" != "true" && "${value}" != "false" ]]; then + planner_valid=false + fi + done + if [[ "${BASELINE_AVAILABLE}" == "true" && + ( -z "${BASELINE_RUN_ID}" || -z "${BASELINE_SHA}" ) ]]; then + planner_valid=false + fi + fi + + # Non-PR events produce the complete trusted artifact set. PRs also + # run everything when the trusted base artifact inventory is absent + # or the planner did not emit a complete boolean result. + if [[ "${IS_PR}" != "true" || + "${BASELINE_AVAILABLE}" != "true" || + "${planner_valid}" != "true" ]]; then bindings=true core=true pathfinder=true python_meta=true test_helpers=true shared=true + build_bindings=true + build_core=true + build_pathfinder=true + build_python=true + test_bindings=true + test_core=true + test_pathfinder=true + test_python=true + baseline_run_id="" + baseline_sha="" else bindings="${BINDINGS}" core="${CORE}" @@ -189,32 +325,18 @@ jobs: python_meta="${PYTHON_META}" test_helpers="${TEST_HELPERS}" shared="${SHARED}" + build_bindings="${BUILD_BINDINGS}" + build_core="${BUILD_CORE}" + build_pathfinder="${BUILD_PATHFINDER}" + build_python="${BUILD_PYTHON}" + test_bindings="${TEST_BINDINGS}" + test_core="${TEST_CORE}" + test_pathfinder="${TEST_PATHFINDER}" + test_python="${TEST_PYTHON}" + baseline_run_id="${BASELINE_RUN_ID}" + baseline_sha="${BASELINE_SHA}" fi - or_flag() { - for v in "$@"; do - if [[ "${v}" == "true" ]]; then - echo "true" - return - fi - done - echo "false" - } - - # Build gating: pathfinder change forces rebuild of bindings and - # core; bindings change forces rebuild of core. shared changes force - # a full rebuild. - build_pathfinder="$(or_flag "${shared}" "${pathfinder}")" - build_bindings="$(or_flag "${shared}" "${pathfinder}" "${bindings}")" - build_core="$(or_flag "${shared}" "${pathfinder}" "${bindings}" "${core}")" - - # Test gating: tests for a module must run whenever that module, any - # of its runtime dependencies, the shared test helper package, or - # shared infra changes. pathfinder tests are cheap and always run. - test_pathfinder=true - test_bindings="$(or_flag "${shared}" "${pathfinder}" "${bindings}" "${test_helpers}")" - test_core="$(or_flag "${shared}" "${pathfinder}" "${bindings}" "${core}" "${test_helpers}")" - { echo "bindings=${bindings}" echo "core=${core}" @@ -225,11 +347,27 @@ jobs: echo "build_bindings=${build_bindings}" echo "build_core=${build_core}" echo "build_pathfinder=${build_pathfinder}" + echo "build_python=${build_python}" echo "test_bindings=${test_bindings}" echo "test_core=${test_core}" echo "test_pathfinder=${test_pathfinder}" + echo "test_python=${test_python}" + echo "baseline_run_id=${baseline_run_id}" + echo "baseline_sha=${baseline_sha}" } >> "$GITHUB_OUTPUT" + { + echo + echo "### Effective package plan" + echo + echo "| Package | Build | Test |" + echo "| --- | --- | --- |" + echo "| cuda-pathfinder | ${build_pathfinder} | ${test_pathfinder} |" + echo "| cuda-bindings | ${build_bindings} | ${test_bindings} |" + echo "| cuda-core | ${build_core} | ${test_core} |" + echo "| cuda-python | ${build_python} | ${test_python} |" + } >> "$GITHUB_STEP_SUMMARY" + api-check-core-vs-release: name: API check (cuda_core vs. latest release) if: >- @@ -314,15 +452,17 @@ jobs: merge-base: ${{ needs.detect-changes.outputs.pr_merge_base }} # NOTE: Build jobs are intentionally split by platform rather than using a single - # matrix. This allows each test job to depend only on its corresponding build, - # so faster platforms can proceed through build & test without waiting for slower - # ones. Keep these job definitions textually identical except for: + # matrix. This lets each test job consume its platform-specific artifacts as + # soon as they are ready. ARM64 and Windows tests also wait for linux-64, + # which produces the universal pathfinder and cuda-python wheels. Keep these + # job definitions textually identical except for: # - host-platform value # - if: condition (build-linux-64 omits doc-only check since it's needed for docs) build-linux-64: needs: - ci-vars - should-skip + - detect-changes strategy: fail-fast: false matrix: @@ -330,50 +470,105 @@ jobs: - linux-64 name: Build ${{ matrix.host-platform }}, CUDA ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} if: ${{ github.repository_owner == 'nvidia' && !fromJSON(needs.should-skip.outputs.skip) }} + permissions: + actions: read + contents: read secrets: inherit uses: ./.github/workflows/build-wheel.yml with: host-platform: ${{ matrix.host-platform }} cuda-version: ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} prev-cuda-version: ${{ needs.ci-vars.outputs.CUDA_PREV_BUILD_VER }} + build-pathfinder: ${{ fromJSON(needs.detect-changes.outputs.build_pathfinder) }} + build-bindings: ${{ fromJSON(needs.detect-changes.outputs.build_bindings) }} + build-core: ${{ fromJSON(needs.detect-changes.outputs.build_core) }} + build-python: ${{ fromJSON(needs.detect-changes.outputs.build_python) }} + test-bindings: ${{ fromJSON(needs.detect-changes.outputs.test_bindings) }} + test-core: ${{ fromJSON(needs.detect-changes.outputs.test_core) }} + baseline-run-id: ${{ needs.detect-changes.outputs.baseline_run_id }} + baseline-sha: ${{ needs.detect-changes.outputs.baseline_sha }} # See build-linux-64 for why build jobs are split by platform. build-linux-aarch64: needs: - ci-vars - should-skip + - detect-changes strategy: fail-fast: false matrix: host-platform: - linux-aarch64 name: Build ${{ matrix.host-platform }}, CUDA ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} - if: ${{ github.repository_owner == 'nvidia' && !fromJSON(needs.should-skip.outputs.skip) && !fromJSON(needs.should-skip.outputs.doc-only) }} + if: ${{ github.repository_owner == 'nvidia' && + !fromJSON(needs.should-skip.outputs.skip) && + !fromJSON(needs.should-skip.outputs.doc-only) && + (fromJSON(needs.detect-changes.outputs.build_pathfinder) || + fromJSON(needs.detect-changes.outputs.build_bindings) || + fromJSON(needs.detect-changes.outputs.build_core) || + fromJSON(needs.detect-changes.outputs.build_python) || + fromJSON(needs.detect-changes.outputs.test_pathfinder) || + fromJSON(needs.detect-changes.outputs.test_bindings) || + fromJSON(needs.detect-changes.outputs.test_core) || + fromJSON(needs.detect-changes.outputs.test_python)) }} + permissions: + actions: read + contents: read secrets: inherit uses: ./.github/workflows/build-wheel.yml with: host-platform: ${{ matrix.host-platform }} cuda-version: ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} prev-cuda-version: ${{ needs.ci-vars.outputs.CUDA_PREV_BUILD_VER }} + build-pathfinder: ${{ fromJSON(needs.detect-changes.outputs.build_pathfinder) }} + build-bindings: ${{ fromJSON(needs.detect-changes.outputs.build_bindings) }} + build-core: ${{ fromJSON(needs.detect-changes.outputs.build_core) }} + build-python: ${{ fromJSON(needs.detect-changes.outputs.build_python) }} + test-bindings: ${{ fromJSON(needs.detect-changes.outputs.test_bindings) }} + test-core: ${{ fromJSON(needs.detect-changes.outputs.test_core) }} + baseline-run-id: ${{ needs.detect-changes.outputs.baseline_run_id }} + baseline-sha: ${{ needs.detect-changes.outputs.baseline_sha }} # See build-linux-64 for why build jobs are split by platform. build-windows: needs: - ci-vars - should-skip + - detect-changes strategy: fail-fast: false matrix: host-platform: - win-64 name: Build ${{ matrix.host-platform }}, CUDA ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} - if: ${{ github.repository_owner == 'nvidia' && !fromJSON(needs.should-skip.outputs.skip) && !fromJSON(needs.should-skip.outputs.doc-only) }} + if: ${{ github.repository_owner == 'nvidia' && + !fromJSON(needs.should-skip.outputs.skip) && + !fromJSON(needs.should-skip.outputs.doc-only) && + (fromJSON(needs.detect-changes.outputs.build_pathfinder) || + fromJSON(needs.detect-changes.outputs.build_bindings) || + fromJSON(needs.detect-changes.outputs.build_core) || + fromJSON(needs.detect-changes.outputs.build_python) || + fromJSON(needs.detect-changes.outputs.test_pathfinder) || + fromJSON(needs.detect-changes.outputs.test_bindings) || + fromJSON(needs.detect-changes.outputs.test_core) || + fromJSON(needs.detect-changes.outputs.test_python)) }} + permissions: + actions: read + contents: read secrets: inherit uses: ./.github/workflows/build-wheel.yml with: host-platform: ${{ matrix.host-platform }} cuda-version: ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} prev-cuda-version: ${{ needs.ci-vars.outputs.CUDA_PREV_BUILD_VER }} + build-pathfinder: ${{ fromJSON(needs.detect-changes.outputs.build_pathfinder) }} + build-bindings: ${{ fromJSON(needs.detect-changes.outputs.build_bindings) }} + build-core: ${{ fromJSON(needs.detect-changes.outputs.build_core) }} + build-python: ${{ fromJSON(needs.detect-changes.outputs.build_python) }} + test-bindings: ${{ fromJSON(needs.detect-changes.outputs.test_bindings) }} + test-core: ${{ fromJSON(needs.detect-changes.outputs.test_core) }} + baseline-run-id: ${{ needs.detect-changes.outputs.baseline_run_id }} + baseline-sha: ${{ needs.detect-changes.outputs.baseline_sha }} # NOTE: test-sdist jobs are split by platform (mirroring build-* and test-wheel-*) # so platform-specific sources (e.g. cuda_bindings/*_windows.pyx selected by @@ -385,26 +580,57 @@ jobs: needs: - ci-vars - should-skip + - detect-changes + - build-linux-64 name: Test sdist linux-64 - if: ${{ github.repository_owner == 'nvidia' && !fromJSON(needs.should-skip.outputs.skip) && !fromJSON(needs.should-skip.outputs.doc-only) }} + if: ${{ github.repository_owner == 'nvidia' && + !fromJSON(needs.should-skip.outputs.skip) && + !fromJSON(needs.should-skip.outputs.doc-only) && + (fromJSON(needs.detect-changes.outputs.build_pathfinder) || + fromJSON(needs.detect-changes.outputs.build_bindings) || + fromJSON(needs.detect-changes.outputs.build_core) || + fromJSON(needs.detect-changes.outputs.build_python)) }} + permissions: + actions: read + contents: read secrets: inherit uses: ./.github/workflows/test-sdist-linux.yml with: host-platform: linux-64 cuda-version: ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} + build-pathfinder: ${{ fromJSON(needs.detect-changes.outputs.build_pathfinder) }} + build-bindings: ${{ fromJSON(needs.detect-changes.outputs.build_bindings) }} + build-core: ${{ fromJSON(needs.detect-changes.outputs.build_core) }} + build-python: ${{ fromJSON(needs.detect-changes.outputs.build_python) }} # See test-sdist-linux for why sdist test jobs are split by platform. test-sdist-windows: needs: - ci-vars - should-skip + - detect-changes + - build-linux-64 + - build-windows name: Test sdist win-64 - if: ${{ github.repository_owner == 'nvidia' && !fromJSON(needs.should-skip.outputs.skip) && !fromJSON(needs.should-skip.outputs.doc-only) }} + if: ${{ github.repository_owner == 'nvidia' && + !fromJSON(needs.should-skip.outputs.skip) && + !fromJSON(needs.should-skip.outputs.doc-only) && + (fromJSON(needs.detect-changes.outputs.build_pathfinder) || + fromJSON(needs.detect-changes.outputs.build_bindings) || + fromJSON(needs.detect-changes.outputs.build_core) || + fromJSON(needs.detect-changes.outputs.build_python)) }} + permissions: + actions: read + contents: read secrets: inherit uses: ./.github/workflows/test-sdist-windows.yml with: host-platform: win-64 cuda-version: ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} + build-pathfinder: ${{ fromJSON(needs.detect-changes.outputs.build_pathfinder) }} + build-bindings: ${{ fromJSON(needs.detect-changes.outputs.build_bindings) }} + build-core: ${{ fromJSON(needs.detect-changes.outputs.build_core) }} + build-python: ${{ fromJSON(needs.detect-changes.outputs.build_python) }} # NOTE: Test jobs are split by platform for the same reason as build jobs (see # build-linux-64). Keep these job definitions textually identical except for: @@ -418,8 +644,14 @@ jobs: host-platform: - linux-64 name: Test ${{ matrix.host-platform }} - if: ${{ github.repository_owner == 'nvidia' && !fromJSON(needs.should-skip.outputs.doc-only) }} + if: ${{ github.repository_owner == 'nvidia' && + !fromJSON(needs.should-skip.outputs.doc-only) && + (fromJSON(needs.detect-changes.outputs.test_pathfinder) || + fromJSON(needs.detect-changes.outputs.test_bindings) || + fromJSON(needs.detect-changes.outputs.test_core) || + fromJSON(needs.detect-changes.outputs.test_python)) }} permissions: + actions: read contents: read # This is required for actions/checkout needs: - ci-vars @@ -433,7 +665,10 @@ jobs: host-platform: ${{ matrix.host-platform }} build-ctk-ver: ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} nruns: ${{ (github.event_name == 'schedule' && 5) || 1}} + test-pathfinder: ${{ fromJSON(needs.detect-changes.outputs.test_pathfinder) }} test-bindings: ${{ fromJSON(needs.detect-changes.outputs.test_bindings) }} + test-core: ${{ fromJSON(needs.detect-changes.outputs.test_core) }} + test-python: ${{ fromJSON(needs.detect-changes.outputs.test_python) }} # See test-linux-64 for why test jobs are split by platform. test-linux-aarch64: @@ -443,13 +678,20 @@ jobs: host-platform: - linux-aarch64 name: Test ${{ matrix.host-platform }} - if: ${{ github.repository_owner == 'nvidia' && !fromJSON(needs.should-skip.outputs.doc-only) }} + if: ${{ github.repository_owner == 'nvidia' && + !fromJSON(needs.should-skip.outputs.doc-only) && + (fromJSON(needs.detect-changes.outputs.test_pathfinder) || + fromJSON(needs.detect-changes.outputs.test_bindings) || + fromJSON(needs.detect-changes.outputs.test_core) || + fromJSON(needs.detect-changes.outputs.test_python)) }} permissions: + actions: read contents: read # This is required for actions/checkout needs: - ci-vars - should-skip - detect-changes + - build-linux-64 - build-linux-aarch64 secrets: inherit uses: ./.github/workflows/test-wheel-linux.yml @@ -458,7 +700,10 @@ jobs: host-platform: ${{ matrix.host-platform }} build-ctk-ver: ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} nruns: ${{ (github.event_name == 'schedule' && 5) || 1}} + test-pathfinder: ${{ fromJSON(needs.detect-changes.outputs.test_pathfinder) }} test-bindings: ${{ fromJSON(needs.detect-changes.outputs.test_bindings) }} + test-core: ${{ fromJSON(needs.detect-changes.outputs.test_core) }} + test-python: ${{ fromJSON(needs.detect-changes.outputs.test_python) }} # See test-linux-64 for why test jobs are split by platform. test-windows: @@ -468,13 +713,20 @@ jobs: host-platform: - win-64 name: Test ${{ matrix.host-platform }} - if: ${{ github.repository_owner == 'nvidia' && !fromJSON(needs.should-skip.outputs.doc-only) }} + if: ${{ github.repository_owner == 'nvidia' && + !fromJSON(needs.should-skip.outputs.doc-only) && + (fromJSON(needs.detect-changes.outputs.test_pathfinder) || + fromJSON(needs.detect-changes.outputs.test_bindings) || + fromJSON(needs.detect-changes.outputs.test_core) || + fromJSON(needs.detect-changes.outputs.test_python)) }} permissions: + actions: read contents: read # This is required for actions/checkout needs: - ci-vars - should-skip - detect-changes + - build-linux-64 - build-windows secrets: inherit uses: ./.github/workflows/test-wheel-windows.yml @@ -483,7 +735,10 @@ jobs: host-platform: ${{ matrix.host-platform }} build-ctk-ver: ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} nruns: ${{ (github.event_name == 'schedule' && 5) || 1}} + test-pathfinder: ${{ fromJSON(needs.detect-changes.outputs.test_pathfinder) }} test-bindings: ${{ fromJSON(needs.detect-changes.outputs.test_bindings) }} + test-core: ${{ fromJSON(needs.detect-changes.outputs.test_core) }} + test-python: ${{ fromJSON(needs.detect-changes.outputs.test_python) }} doc: name: Docs @@ -538,13 +793,19 @@ jobs: if: always() runs-on: ubuntu-latest needs: + - ci-vars - should-skip - detect-changes + - build-linux-64 + - build-linux-aarch64 + - build-windows - test-sdist-linux - test-sdist-windows - test-linux-64 - test-linux-aarch64 - test-windows + - api-check-core-vs-release + - api-check-core-vs-base - doc - precommit-windows steps: @@ -565,6 +826,16 @@ jobs: fi doc_only="${{ needs.should-skip.outputs.doc-only }}" + build_selected="${{ needs.detect-changes.outputs.build_pathfinder == 'true' || + needs.detect-changes.outputs.build_bindings == 'true' || + needs.detect-changes.outputs.build_core == 'true' || + needs.detect-changes.outputs.build_python == 'true' }}" + test_selected="${{ needs.detect-changes.outputs.test_pathfinder == 'true' || + needs.detect-changes.outputs.test_bindings == 'true' || + needs.detect-changes.outputs.test_core == 'true' || + needs.detect-changes.outputs.test_python == 'true' }}" + core_changed="${{ needs.detect-changes.outputs.core == 'true' }}" + is_pr="${{ startsWith(github.ref_name, 'pull-request/') }}" status="success" check_result() { name=$1; expected=$2; result=$3 @@ -575,18 +846,47 @@ jobs: fi } - # always expected to succeed (even in [doc-only] mode) - check_result "should-skip" "success" "${{ needs.should-skip.result }}" - check_result "detect-changes" "success" "${{ needs.detect-changes.result }}" - check_result "doc" "success" "${{ needs.doc.result }}" + # Control jobs, the universal linux build, docs, and Windows + # pre-commit checks always run. + check_result "ci-vars" "success" "${{ needs.ci-vars.result }}" + check_result "should-skip" "success" "${{ needs.should-skip.result }}" + check_result "detect-changes" "success" "${{ needs.detect-changes.result }}" + check_result "build-linux-64" "success" "${{ needs.build-linux-64.result }}" + check_result "doc" "success" "${{ needs.doc.result }}" check_result "precommit-windows" "success" "${{ needs.precommit-windows.result }}" - # [doc-only] flips these from 'success' to 'skipped' - if [[ "$doc_only" == "true" ]]; then expected="skipped"; else expected="success"; fi + # Platform builds run whenever any package needs to be built or tested. + expected="skipped" + if [[ "$doc_only" != "true" && + ( "$build_selected" == "true" || "$test_selected" == "true" ) ]]; then + expected="success" + fi + check_result "build-linux-aarch64" "$expected" "${{ needs.build-linux-aarch64.result }}" + check_result "build-windows" "$expected" "${{ needs.build-windows.result }}" + + # Sdist and wheel tests are independently gated by the effective plan. + expected="skipped" + if [[ "$doc_only" != "true" && "$build_selected" == "true" ]]; then + expected="success" + fi check_result "test-sdist-linux" "$expected" "${{ needs.test-sdist-linux.result }}" check_result "test-sdist-windows" "$expected" "${{ needs.test-sdist-windows.result }}" + + expected="skipped" + if [[ "$doc_only" != "true" && "$test_selected" == "true" ]]; then + expected="success" + fi check_result "test-linux-64" "$expected" "${{ needs.test-linux-64.result }}" check_result "test-linux-aarch64" "$expected" "${{ needs.test-linux-aarch64.result }}" check_result "test-windows" "$expected" "${{ needs.test-windows.result }}" + # API compatibility checks only run for cuda_core source changes. + expected="skipped" + if [[ "$core_changed" == "true" ]]; then expected="success"; fi + check_result "api-check-core-vs-release" "$expected" "${{ needs.api-check-core-vs-release.result }}" + + expected="skipped" + if [[ "$is_pr" == "true" && "$core_changed" == "true" ]]; then expected="success"; fi + check_result "api-check-core-vs-base" "$expected" "${{ needs.api-check-core-vs-base.result }}" + [[ "$status" == "success" ]] diff --git a/ci/tools/compute_ci_plan.py b/ci/tools/compute_ci_plan.py new file mode 100644 index 00000000000..7095514b73e --- /dev/null +++ b/ci/tools/compute_ci_plan.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 + +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Compute the package build and test closure for a set of changed paths.""" + +from __future__ import annotations + +import argparse +from pathlib import Path + +PACKAGES = { + "cuda_pathfinder": "pathfinder", + "cuda_bindings": "bindings", + "cuda_core": "core", + "cuda_python": "python", +} + +SHARED_PREFIXES = ( + ".github/", + "ci/", + "scripts/", + "toolshed/", +) + +SHARED_FILES = { + ".pre-commit-config.yaml", + "conftest.py", + "pixi.lock", + "pixi.toml", + "pytest.ini", + "ruff.toml", +} + +KNOWN_REPOSITORY_FILES = { + ".git-blame-ignore-revs", + ".gitignore", + "AGENTS.md", + "CHANGELOG.md", + "CODE_OF_CONDUCT.md", + "CONTRIBUTING.md", + "LICENSE", + "README.md", + "SECURITY.md", +} + + +def _bool(value: bool) -> str: + return str(value).lower() + + +def _read_paths(path: Path) -> list[str]: + return [value.decode("utf-8", errors="surrogateescape") for value in path.read_bytes().split(b"\0") if value] + + +def compute_plan(paths: list[str]) -> dict[str, bool]: + source = dict.fromkeys(PACKAGES.values(), False) + tests = dict.fromkeys(PACKAGES.values(), False) + docs = False + test_helpers = False + shared = False + unknown = False + + for path in paths: + package_dir, separator, relative_path = path.partition("/") + package = PACKAGES.get(package_dir) + if package is not None and separator: + if relative_path.startswith("docs/"): + docs = True + elif relative_path.startswith(("tests/", "examples/")): + tests[package] = True + else: + source[package] = True + continue + + if path.startswith("cuda_python_test_helpers/"): + test_helpers = True + elif path.startswith("benchmarks/cuda_bindings/"): + tests["bindings"] = True + elif path.startswith(SHARED_PREFIXES) or path in SHARED_FILES: + shared = True + elif path in KNOWN_REPOSITORY_FILES: + # Repository policy and prose files do not affect package artifacts. + continue + else: + unknown = True + + full = shared or unknown + + build_pathfinder = full or source["pathfinder"] + # Development cuda-python wheels exactly pin cuda-bindings, so a + # metapackage change needs a matching bindings artifact for its smoke test. + build_bindings = full or source["pathfinder"] or source["bindings"] or source["python"] + build_core = full or source["pathfinder"] or source["bindings"] or source["core"] + # A core-only change can reuse the baseline cuda-python wheel: rebuilding + # it would also require rebuilding the exact-version cuda-bindings pin. + build_python = full or source["pathfinder"] or source["bindings"] or source["python"] + + test_pathfinder = full or source["pathfinder"] or tests["pathfinder"] + test_bindings = full or source["pathfinder"] or source["bindings"] or tests["bindings"] or test_helpers + test_core = full or source["pathfinder"] or source["bindings"] or source["core"] or tests["core"] or test_helpers + test_python = ( + full or source["pathfinder"] or source["bindings"] or source["core"] or source["python"] or tests["python"] + ) + + return { + "shared": shared, + "unknown": unknown, + "docs": docs, + "test_helpers": test_helpers, + "pathfinder_source": source["pathfinder"], + "bindings_source": source["bindings"], + "core_source": source["core"], + "python_source": source["python"], + "pathfinder_tests": tests["pathfinder"], + "bindings_tests": tests["bindings"], + "core_tests": tests["core"], + "python_tests": tests["python"], + "build_pathfinder": build_pathfinder, + "build_bindings": build_bindings, + "build_core": build_core, + "build_python": build_python, + "test_pathfinder": test_pathfinder, + "test_bindings": test_bindings, + "test_core": test_core, + "test_python": test_python, + } + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument( + "paths_file", + type=Path, + help="NUL-separated changed-path list produced by git diff --name-only -z", + ) + args = parser.parse_args() + + plan = compute_plan(_read_paths(args.paths_file)) + for key, value in plan.items(): + print(f"{key}={_bool(value)}") + + +if __name__ == "__main__": + main() From da7ebee702759b45c0a81fa0c1a9f7f233fbf874 Mon Sep 17 00:00:00 2001 From: Keith Kraus Date: Thu, 13 Aug 2026 22:45:31 -0400 Subject: [PATCH 2/4] ci: use paths-filter for selective CI planning --- .github/workflows/ci.yml | 307 ++++++++++++++++++------------------ ci/tools/compute_ci_plan.py | 147 ----------------- 2 files changed, 156 insertions(+), 298 deletions(-) delete mode 100644 ci/tools/compute_ci_plan.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0be7f9445b5..16ab65b3db2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -86,7 +86,7 @@ jobs: # downstream module. A change to cuda_bindings forces rebuild of cuda_core. # A change to cuda_core alone skips rebuilding/retesting cuda_bindings and # cuda_pathfinder, but still retests the downstream cuda-python metapackage. - # CI/planner changes are shared by design, so this implementation runs the + # CI/filter changes are shared by design, so this implementation runs the # full pipeline; exercise selective cases in follow-up package-only PRs. # On push to main, tag refs, schedule, or workflow_dispatch events we # unconditionally run everything because there is no meaningful "changed @@ -98,43 +98,98 @@ jobs: contents: read pull-requests: read outputs: - bindings: ${{ steps.compose.outputs.bindings }} - core: ${{ steps.compose.outputs.core }} - pathfinder: ${{ steps.compose.outputs.pathfinder }} - python_meta: ${{ steps.compose.outputs.python_meta }} - test_helpers: ${{ steps.compose.outputs.test_helpers }} - shared: ${{ steps.compose.outputs.shared }} - build_bindings: ${{ steps.compose.outputs.build_bindings }} - build_core: ${{ steps.compose.outputs.build_core }} - build_pathfinder: ${{ steps.compose.outputs.build_pathfinder }} - build_python: ${{ steps.compose.outputs.build_python }} - test_bindings: ${{ steps.compose.outputs.test_bindings }} - test_core: ${{ steps.compose.outputs.test_core }} - test_pathfinder: ${{ steps.compose.outputs.test_pathfinder }} - pr_merge_base: ${{ steps.filter.outputs.merge_base }} - test_python: ${{ steps.compose.outputs.test_python }} - baseline_run_id: ${{ steps.compose.outputs.baseline_run_id }} - baseline_sha: ${{ steps.compose.outputs.baseline_sha }} + # Missing base artifacts or a skipped path filter fail open to the full pipeline. + core: >- + ${{ steps.baseline.outputs.available != 'true' || + steps.filter.outputs.changes == '' || + steps.filter.outputs.core_source == 'true' }} + build_pathfinder: >- + ${{ steps.baseline.outputs.available != 'true' || + steps.filter.outputs.changes == '' || + steps.filter.outputs.force_all == 'true' || + steps.filter.outputs.pathfinder_source == 'true' }} + build_bindings: >- + ${{ steps.baseline.outputs.available != 'true' || + steps.filter.outputs.changes == '' || + steps.filter.outputs.force_all == 'true' || + steps.filter.outputs.pathfinder_source == 'true' || + steps.filter.outputs.bindings_source == 'true' || + steps.filter.outputs.python_source == 'true' }} + build_core: >- + ${{ steps.baseline.outputs.available != 'true' || + steps.filter.outputs.changes == '' || + steps.filter.outputs.force_all == 'true' || + steps.filter.outputs.pathfinder_source == 'true' || + steps.filter.outputs.bindings_source == 'true' || + steps.filter.outputs.core_source == 'true' }} + build_python: >- + ${{ steps.baseline.outputs.available != 'true' || + steps.filter.outputs.changes == '' || + steps.filter.outputs.force_all == 'true' || + steps.filter.outputs.pathfinder_source == 'true' || + steps.filter.outputs.bindings_source == 'true' || + steps.filter.outputs.python_source == 'true' }} + test_pathfinder: >- + ${{ steps.baseline.outputs.available != 'true' || + steps.filter.outputs.changes == '' || + steps.filter.outputs.force_all == 'true' || + steps.filter.outputs.pathfinder_source == 'true' || + steps.filter.outputs.pathfinder_tests == 'true' }} + test_bindings: >- + ${{ steps.baseline.outputs.available != 'true' || + steps.filter.outputs.changes == '' || + steps.filter.outputs.force_all == 'true' || + steps.filter.outputs.pathfinder_source == 'true' || + steps.filter.outputs.bindings_source == 'true' || + steps.filter.outputs.bindings_tests == 'true' || + steps.filter.outputs.test_helpers == 'true' }} + test_core: >- + ${{ steps.baseline.outputs.available != 'true' || + steps.filter.outputs.changes == '' || + steps.filter.outputs.force_all == 'true' || + steps.filter.outputs.pathfinder_source == 'true' || + steps.filter.outputs.bindings_source == 'true' || + steps.filter.outputs.core_source == 'true' || + steps.filter.outputs.core_tests == 'true' || + steps.filter.outputs.test_helpers == 'true' }} + pr_merge_base: ${{ steps.merge-base.outputs.sha }} + test_python: >- + ${{ steps.baseline.outputs.available != 'true' || + steps.filter.outputs.changes == '' || + steps.filter.outputs.force_all == 'true' || + steps.filter.outputs.pathfinder_source == 'true' || + steps.filter.outputs.bindings_source == 'true' || + steps.filter.outputs.core_source == 'true' || + steps.filter.outputs.python_source == 'true' || + steps.filter.outputs.python_tests == 'true' }} + baseline_run_id: >- + ${{ steps.filter.outputs.changes != '' && + steps.baseline.outputs.available == 'true' && + steps.baseline.outputs.run_id || '' }} + baseline_sha: >- + ${{ steps.filter.outputs.changes != '' && + steps.baseline.outputs.available == 'true' && + steps.baseline.outputs.sha || '' }} steps: - name: Checkout repository uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: - # Treeless clone: commit graph is needed for `git merge-base` and - # `git diff --name-only` below, but historical blobs aren't. + # Treeless clone: the commit graph is needed to resolve the PR merge + # base and classify its changed paths, but historical blobs aren't. fetch-depth: 0 filter: blob:none # copy-pr-bot pushes every PR (whether it targets main or a backport # branch such as 12.9.x) to pull-request/, so the base branch # cannot be inferred from github.ref_name. Look it up via the - # upstream PR metadata so the diff below is rooted at the right place. + # upstream PR metadata so change detection is rooted at the right place. - name: Resolve PR base branch id: pr-info if: ${{ startsWith(github.ref_name, 'pull-request/') }} uses: nv-gha-runners/get-pr-info@main - - name: Detect changed paths - id: filter + - name: Resolve PR merge base + id: merge-base if: ${{ startsWith(github.ref_name, 'pull-request/') }} env: # GitHub Actions evaluates step-level `env:` expressions eagerly — @@ -153,25 +208,83 @@ jobs: exit 1 fi - # Diff against the merge base with the PR's actual target branch. - # Disabling rename detection reports both sides of a cross-package - # move, which prevents the source package from being skipped. base=$(git merge-base HEAD "origin/${BASE_REF}") - git diff --no-renames --name-only -z "$base"...HEAD > changed-paths - python ci/tools/compute_ci_plan.py changed-paths >> "$GITHUB_OUTPUT" - echo "merge_base=${base}" >> "$GITHUB_OUTPUT" + echo "sha=${base}" >> "$GITHUB_OUTPUT" - { - echo "### Selective CI changed paths" - echo - tr '\0' '\n' < changed-paths | sed 's/^/- `/' | sed 's/$/`/' - } >> "$GITHUB_STEP_SUMMARY" + - name: Classify changed paths + id: filter + if: ${{ startsWith(github.ref_name, 'pull-request/') }} + uses: dorny/paths-filter@ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d # v4.0.3 + with: + # The workflow runs on copy-pr-bot push branches, so compare the + # checked-out PR head against the base resolved from PR metadata. + base: ${{ steps.merge-base.outputs.sha }} + ref: ${{ github.sha }} + token: '' + predicate-quantifier: some-with-excludes + filters: | + pathfinder_source: + - 'cuda_pathfinder/**' + - '!cuda_pathfinder/docs/**' + - '!cuda_pathfinder/tests/**' + - '!cuda_pathfinder/examples/**' + bindings_source: + - 'cuda_bindings/**' + - '!cuda_bindings/docs/**' + - '!cuda_bindings/tests/**' + - '!cuda_bindings/examples/**' + core_source: + - 'cuda_core/**' + - '!cuda_core/docs/**' + - '!cuda_core/tests/**' + - '!cuda_core/examples/**' + python_source: + - 'cuda_python/**' + - '!cuda_python/docs/**' + - '!cuda_python/tests/**' + - '!cuda_python/examples/**' + pathfinder_tests: + - 'cuda_pathfinder/tests/**' + - 'cuda_pathfinder/examples/**' + bindings_tests: + - 'cuda_bindings/tests/**' + - 'cuda_bindings/examples/**' + - 'benchmarks/cuda_bindings/**' + core_tests: + - 'cuda_core/tests/**' + - 'cuda_core/examples/**' + python_tests: + - 'cuda_python/tests/**' + - 'cuda_python/examples/**' + test_helpers: + - 'cuda_python_test_helpers/**' + # Shared infrastructure and unknown paths run the full pipeline. + # Exclude only paths whose narrower behavior is defined above or + # repository policy/prose files known not to affect artifacts. + force_all: + - '**' + - '!cuda_pathfinder/**' + - '!cuda_bindings/**' + - '!cuda_core/**' + - '!cuda_python/**' + - '!cuda_python_test_helpers/**' + - '!benchmarks/cuda_bindings/**' + - '!.git-blame-ignore-revs' + - '!.gitignore' + - '!AGENTS.md' + - '!CHANGELOG.md' + - '!CODE_OF_CONDUCT.md' + - '!CONTRIBUTING.md' + - '!LICENSE' + - '!README.md' + - '!SECURITY.md' - name: Resolve reusable base artifacts id: baseline if: ${{ startsWith(github.ref_name, 'pull-request/') }} env: BASE_REF: ${{ steps.pr-info.outputs.pr-info && fromJSON(steps.pr-info.outputs.pr-info).base.ref || '' }} + MERGE_BASE: ${{ steps.merge-base.outputs.sha }} GH_TOKEN: ${{ github.token }} run: | set -uo pipefail @@ -186,7 +299,10 @@ jobs: unavailable fi - merge_base=$(git merge-base HEAD "origin/${BASE_REF}") + merge_base="${MERGE_BASE}" + if [[ -z "${merge_base}" ]]; then + unavailable + fi if ! runs=$(gh run list \ --repo "${{ github.repository }}" \ --branch "${BASE_REF}" \ @@ -256,118 +372,6 @@ jobs: echo "Reusable artifacts: run \`${run_id}\` at \`${baseline_sha}\` on \`${BASE_REF}\`." } >> "$GITHUB_STEP_SUMMARY" - - name: Compose gating outputs - id: compose - env: - IS_PR: ${{ startsWith(github.ref_name, 'pull-request/') }} - BASELINE_AVAILABLE: ${{ steps.baseline.outputs.available || 'false' }} - BASELINE_RUN_ID: ${{ steps.baseline.outputs.run_id }} - BASELINE_SHA: ${{ steps.baseline.outputs.sha }} - BINDINGS: ${{ steps.filter.outputs.bindings_source }} - CORE: ${{ steps.filter.outputs.core_source }} - PATHFINDER: ${{ steps.filter.outputs.pathfinder_source }} - PYTHON_META: ${{ steps.filter.outputs.python_source }} - TEST_HELPERS: ${{ steps.filter.outputs.test_helpers }} - SHARED: ${{ steps.filter.outputs.shared }} - BUILD_BINDINGS: ${{ steps.filter.outputs.build_bindings }} - BUILD_CORE: ${{ steps.filter.outputs.build_core }} - BUILD_PATHFINDER: ${{ steps.filter.outputs.build_pathfinder }} - BUILD_PYTHON: ${{ steps.filter.outputs.build_python }} - TEST_BINDINGS: ${{ steps.filter.outputs.test_bindings }} - TEST_CORE: ${{ steps.filter.outputs.test_core }} - TEST_PATHFINDER: ${{ steps.filter.outputs.test_pathfinder }} - TEST_PYTHON: ${{ steps.filter.outputs.test_python }} - run: | - set -euxo pipefail - planner_valid=true - if [[ "${IS_PR}" == "true" ]]; then - for value in \ - "${BINDINGS}" "${CORE}" "${PATHFINDER}" "${PYTHON_META}" \ - "${TEST_HELPERS}" "${SHARED}" \ - "${BUILD_BINDINGS}" "${BUILD_CORE}" "${BUILD_PATHFINDER}" "${BUILD_PYTHON}" \ - "${TEST_BINDINGS}" "${TEST_CORE}" "${TEST_PATHFINDER}" "${TEST_PYTHON}"; do - if [[ "${value}" != "true" && "${value}" != "false" ]]; then - planner_valid=false - fi - done - if [[ "${BASELINE_AVAILABLE}" == "true" && - ( -z "${BASELINE_RUN_ID}" || -z "${BASELINE_SHA}" ) ]]; then - planner_valid=false - fi - fi - - # Non-PR events produce the complete trusted artifact set. PRs also - # run everything when the trusted base artifact inventory is absent - # or the planner did not emit a complete boolean result. - if [[ "${IS_PR}" != "true" || - "${BASELINE_AVAILABLE}" != "true" || - "${planner_valid}" != "true" ]]; then - bindings=true - core=true - pathfinder=true - python_meta=true - test_helpers=true - shared=true - build_bindings=true - build_core=true - build_pathfinder=true - build_python=true - test_bindings=true - test_core=true - test_pathfinder=true - test_python=true - baseline_run_id="" - baseline_sha="" - else - bindings="${BINDINGS}" - core="${CORE}" - pathfinder="${PATHFINDER}" - python_meta="${PYTHON_META}" - test_helpers="${TEST_HELPERS}" - shared="${SHARED}" - build_bindings="${BUILD_BINDINGS}" - build_core="${BUILD_CORE}" - build_pathfinder="${BUILD_PATHFINDER}" - build_python="${BUILD_PYTHON}" - test_bindings="${TEST_BINDINGS}" - test_core="${TEST_CORE}" - test_pathfinder="${TEST_PATHFINDER}" - test_python="${TEST_PYTHON}" - baseline_run_id="${BASELINE_RUN_ID}" - baseline_sha="${BASELINE_SHA}" - fi - - { - echo "bindings=${bindings}" - echo "core=${core}" - echo "pathfinder=${pathfinder}" - echo "python_meta=${python_meta}" - echo "test_helpers=${test_helpers}" - echo "shared=${shared}" - echo "build_bindings=${build_bindings}" - echo "build_core=${build_core}" - echo "build_pathfinder=${build_pathfinder}" - echo "build_python=${build_python}" - echo "test_bindings=${test_bindings}" - echo "test_core=${test_core}" - echo "test_pathfinder=${test_pathfinder}" - echo "test_python=${test_python}" - echo "baseline_run_id=${baseline_run_id}" - echo "baseline_sha=${baseline_sha}" - } >> "$GITHUB_OUTPUT" - - { - echo - echo "### Effective package plan" - echo - echo "| Package | Build | Test |" - echo "| --- | --- | --- |" - echo "| cuda-pathfinder | ${build_pathfinder} | ${test_pathfinder} |" - echo "| cuda-bindings | ${build_bindings} | ${test_bindings} |" - echo "| cuda-core | ${build_core} | ${test_core} |" - echo "| cuda-python | ${build_python} | ${test_python} |" - } >> "$GITHUB_STEP_SUMMARY" - api-check-core-vs-release: name: API check (cuda_core vs. latest release) if: >- @@ -834,7 +838,7 @@ jobs: needs.detect-changes.outputs.test_bindings == 'true' || needs.detect-changes.outputs.test_core == 'true' || needs.detect-changes.outputs.test_python == 'true' }}" - core_changed="${{ needs.detect-changes.outputs.core == 'true' }}" + run_core_api_check="${{ needs.detect-changes.outputs.core == 'true' }}" is_pr="${{ startsWith(github.ref_name, 'pull-request/') }}" status="success" check_result() { @@ -880,13 +884,14 @@ jobs: check_result "test-linux-aarch64" "$expected" "${{ needs.test-linux-aarch64.result }}" check_result "test-windows" "$expected" "${{ needs.test-windows.result }}" - # API compatibility checks only run for cuda_core source changes. + # API compatibility checks run for cuda_core source changes and for + # conservative full runs when reusable base artifacts are unavailable. expected="skipped" - if [[ "$core_changed" == "true" ]]; then expected="success"; fi + if [[ "$run_core_api_check" == "true" ]]; then expected="success"; fi check_result "api-check-core-vs-release" "$expected" "${{ needs.api-check-core-vs-release.result }}" expected="skipped" - if [[ "$is_pr" == "true" && "$core_changed" == "true" ]]; then expected="success"; fi + if [[ "$is_pr" == "true" && "$run_core_api_check" == "true" ]]; then expected="success"; fi check_result "api-check-core-vs-base" "$expected" "${{ needs.api-check-core-vs-base.result }}" [[ "$status" == "success" ]] diff --git a/ci/tools/compute_ci_plan.py b/ci/tools/compute_ci_plan.py deleted file mode 100644 index 7095514b73e..00000000000 --- a/ci/tools/compute_ci_plan.py +++ /dev/null @@ -1,147 +0,0 @@ -#!/usr/bin/env python3 - -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# SPDX-License-Identifier: Apache-2.0 - -"""Compute the package build and test closure for a set of changed paths.""" - -from __future__ import annotations - -import argparse -from pathlib import Path - -PACKAGES = { - "cuda_pathfinder": "pathfinder", - "cuda_bindings": "bindings", - "cuda_core": "core", - "cuda_python": "python", -} - -SHARED_PREFIXES = ( - ".github/", - "ci/", - "scripts/", - "toolshed/", -) - -SHARED_FILES = { - ".pre-commit-config.yaml", - "conftest.py", - "pixi.lock", - "pixi.toml", - "pytest.ini", - "ruff.toml", -} - -KNOWN_REPOSITORY_FILES = { - ".git-blame-ignore-revs", - ".gitignore", - "AGENTS.md", - "CHANGELOG.md", - "CODE_OF_CONDUCT.md", - "CONTRIBUTING.md", - "LICENSE", - "README.md", - "SECURITY.md", -} - - -def _bool(value: bool) -> str: - return str(value).lower() - - -def _read_paths(path: Path) -> list[str]: - return [value.decode("utf-8", errors="surrogateescape") for value in path.read_bytes().split(b"\0") if value] - - -def compute_plan(paths: list[str]) -> dict[str, bool]: - source = dict.fromkeys(PACKAGES.values(), False) - tests = dict.fromkeys(PACKAGES.values(), False) - docs = False - test_helpers = False - shared = False - unknown = False - - for path in paths: - package_dir, separator, relative_path = path.partition("/") - package = PACKAGES.get(package_dir) - if package is not None and separator: - if relative_path.startswith("docs/"): - docs = True - elif relative_path.startswith(("tests/", "examples/")): - tests[package] = True - else: - source[package] = True - continue - - if path.startswith("cuda_python_test_helpers/"): - test_helpers = True - elif path.startswith("benchmarks/cuda_bindings/"): - tests["bindings"] = True - elif path.startswith(SHARED_PREFIXES) or path in SHARED_FILES: - shared = True - elif path in KNOWN_REPOSITORY_FILES: - # Repository policy and prose files do not affect package artifacts. - continue - else: - unknown = True - - full = shared or unknown - - build_pathfinder = full or source["pathfinder"] - # Development cuda-python wheels exactly pin cuda-bindings, so a - # metapackage change needs a matching bindings artifact for its smoke test. - build_bindings = full or source["pathfinder"] or source["bindings"] or source["python"] - build_core = full or source["pathfinder"] or source["bindings"] or source["core"] - # A core-only change can reuse the baseline cuda-python wheel: rebuilding - # it would also require rebuilding the exact-version cuda-bindings pin. - build_python = full or source["pathfinder"] or source["bindings"] or source["python"] - - test_pathfinder = full or source["pathfinder"] or tests["pathfinder"] - test_bindings = full or source["pathfinder"] or source["bindings"] or tests["bindings"] or test_helpers - test_core = full or source["pathfinder"] or source["bindings"] or source["core"] or tests["core"] or test_helpers - test_python = ( - full or source["pathfinder"] or source["bindings"] or source["core"] or source["python"] or tests["python"] - ) - - return { - "shared": shared, - "unknown": unknown, - "docs": docs, - "test_helpers": test_helpers, - "pathfinder_source": source["pathfinder"], - "bindings_source": source["bindings"], - "core_source": source["core"], - "python_source": source["python"], - "pathfinder_tests": tests["pathfinder"], - "bindings_tests": tests["bindings"], - "core_tests": tests["core"], - "python_tests": tests["python"], - "build_pathfinder": build_pathfinder, - "build_bindings": build_bindings, - "build_core": build_core, - "build_python": build_python, - "test_pathfinder": test_pathfinder, - "test_bindings": test_bindings, - "test_core": test_core, - "test_python": test_python, - } - - -def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument( - "paths_file", - type=Path, - help="NUL-separated changed-path list produced by git diff --name-only -z", - ) - args = parser.parse_args() - - plan = compute_plan(_read_paths(args.paths_file)) - for key, value in plan.items(): - print(f"{key}={_bool(value)}") - - -if __name__ == "__main__": - main() From 42fe412d683de6f3b2cf7becc72e06d0ddca2d57 Mon Sep 17 00:00:00 2001 From: Keith Kraus Date: Fri, 14 Aug 2026 00:25:40 -0400 Subject: [PATCH 3/4] ci: refine selective path classification --- .github/workflows/ci.yml | 113 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 105 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 16ab65b3db2..9120b2e062f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -86,8 +86,8 @@ jobs: # downstream module. A change to cuda_bindings forces rebuild of cuda_core. # A change to cuda_core alone skips rebuilding/retesting cuda_bindings and # cuda_pathfinder, but still retests the downstream cuda-python metapackage. - # CI/filter changes are shared by design, so this implementation runs the - # full pipeline; exercise selective cases in follow-up package-only PRs. + # Shared build/orchestration changes run the full pipeline; test-only CI + # infrastructure runs every test suite without rebuilding package wheels. # On push to main, tag refs, schedule, or workflow_dispatch events we # unconditionally run everything because there is no meaningful "changed # paths" baseline for those events. @@ -102,6 +102,7 @@ jobs: core: >- ${{ steps.baseline.outputs.available != 'true' || steps.filter.outputs.changes == '' || + steps.filter.outputs.force_all == 'true' || steps.filter.outputs.core_source == 'true' }} build_pathfinder: >- ${{ steps.baseline.outputs.available != 'true' || @@ -133,12 +134,14 @@ jobs: ${{ steps.baseline.outputs.available != 'true' || steps.filter.outputs.changes == '' || steps.filter.outputs.force_all == 'true' || + steps.filter.outputs.all_tests == 'true' || steps.filter.outputs.pathfinder_source == 'true' || steps.filter.outputs.pathfinder_tests == 'true' }} test_bindings: >- ${{ steps.baseline.outputs.available != 'true' || steps.filter.outputs.changes == '' || steps.filter.outputs.force_all == 'true' || + steps.filter.outputs.all_tests == 'true' || steps.filter.outputs.pathfinder_source == 'true' || steps.filter.outputs.bindings_source == 'true' || steps.filter.outputs.bindings_tests == 'true' || @@ -147,6 +150,7 @@ jobs: ${{ steps.baseline.outputs.available != 'true' || steps.filter.outputs.changes == '' || steps.filter.outputs.force_all == 'true' || + steps.filter.outputs.all_tests == 'true' || steps.filter.outputs.pathfinder_source == 'true' || steps.filter.outputs.bindings_source == 'true' || steps.filter.outputs.core_source == 'true' || @@ -157,6 +161,7 @@ jobs: ${{ steps.baseline.outputs.available != 'true' || steps.filter.outputs.changes == '' || steps.filter.outputs.force_all == 'true' || + steps.filter.outputs.all_tests == 'true' || steps.filter.outputs.pathfinder_source == 'true' || steps.filter.outputs.bindings_source == 'true' || steps.filter.outputs.core_source == 'true' || @@ -228,39 +233,79 @@ jobs: - '!cuda_pathfinder/docs/**' - '!cuda_pathfinder/tests/**' - '!cuda_pathfinder/examples/**' + - '!cuda_pathfinder/pixi.lock' + - '!cuda_pathfinder/pixi.toml' + - '!cuda_pathfinder/AGENTS.md' + - '!cuda_pathfinder/CLAUDE.md' bindings_source: - 'cuda_bindings/**' - '!cuda_bindings/docs/**' - '!cuda_bindings/tests/**' - '!cuda_bindings/examples/**' + - '!cuda_bindings/pixi.lock' + - '!cuda_bindings/pixi.toml' + - '!cuda_bindings/AGENTS.md' + - '!cuda_bindings/CLAUDE.md' core_source: - 'cuda_core/**' - '!cuda_core/docs/**' - '!cuda_core/tests/**' - '!cuda_core/examples/**' + - '!cuda_core/pixi.lock' + - '!cuda_core/pixi.toml' + - '!cuda_core/pytest.ini' + - '!cuda_core/AGENTS.md' + - '!cuda_core/CLAUDE.md' python_source: - 'cuda_python/**' - '!cuda_python/docs/**' - '!cuda_python/tests/**' - '!cuda_python/examples/**' + - '!cuda_python/AGENTS.md' + - '!cuda_python/CLAUDE.md' + # cuda_python/README.md is a symlink to this packaging input. + - 'README.md' pathfinder_tests: - 'cuda_pathfinder/tests/**' - 'cuda_pathfinder/examples/**' + - '!cuda_pathfinder/**/AGENTS.md' + - '!cuda_pathfinder/**/CLAUDE.md' bindings_tests: - 'cuda_bindings/tests/**' - 'cuda_bindings/examples/**' - 'benchmarks/cuda_bindings/**' + - '!benchmarks/cuda_bindings/pixi.lock' + - '!benchmarks/cuda_bindings/pixi.toml' + - '!cuda_bindings/**/AGENTS.md' + - '!cuda_bindings/**/CLAUDE.md' core_tests: - 'cuda_core/tests/**' - 'cuda_core/examples/**' + - 'cuda_core/pytest.ini' + - '!cuda_core/**/AGENTS.md' + - '!cuda_core/**/CLAUDE.md' python_tests: - 'cuda_python/tests/**' - 'cuda_python/examples/**' + - '!cuda_python/**/AGENTS.md' + - '!cuda_python/**/CLAUDE.md' test_helpers: - - 'cuda_python_test_helpers/**' + - 'cuda_python_test_helpers/cuda_python_test_helpers/**' + # These files configure or implement wheel tests, but do not + # change any package artifact. + all_tests: + - '.github/workflows/test-wheel-linux.yml' + - '.github/workflows/test-wheel-windows.yml' + - 'ci/test-matrix.yml' + - 'ci/tools/configure_driver_mode.ps1' + - 'ci/tools/guess_latest.sh' + - 'ci/tools/install_gpu_driver.ps1' + - 'ci/tools/install_gpu_driver.sh' + - 'ci/tools/run-tests' + - 'ci/tools/setup-sanitizer' # Shared infrastructure and unknown paths run the full pipeline. - # Exclude only paths whose narrower behavior is defined above or - # repository policy/prose files known not to affect artifacts. + # Exclude paths classified above and paths consumed only by an + # independent or unconditional CI job. force_all: - '**' - '!cuda_pathfinder/**' @@ -269,15 +314,67 @@ jobs: - '!cuda_python/**' - '!cuda_python_test_helpers/**' - '!benchmarks/cuda_bindings/**' - - '!.git-blame-ignore-revs' + - '!benchmarks/cuda_core/**' + - '!.agents/**' + - '!.coveragerc' - '!.gitignore' + - '!.pre-commit-config.yaml' + - '!.spdx-ignore' - '!AGENTS.md' - - '!CHANGELOG.md' - - '!CODE_OF_CONDUCT.md' + - '!CLAUDE.md' - '!CONTRIBUTING.md' - '!LICENSE' - '!README.md' - '!SECURITY.md' + - '!context7.json' + - '!greptile.json' + - '!pixi.lock' + - '!pixi.toml' + - '!pytest.ini' + - '!ruff.toml' + - '!toolshed/**' + - '!.github/ISSUE_TEMPLATE/**' + - '!.github/PULL_REQUEST_TEMPLATE.md' + - '!.github/RELEASE-core.md' + - '!.github/actionlint.yaml' + - '!.github/actions/doc_preview/**' + - '!.github/actions/get_pr_number/**' + - '!.github/copy-pr-bot.yaml' + - '!.github/dependabot.yml' + - '!.github/labeler.yml' + - '!.github/workflows/backport.yml' + - '!.github/workflows/bandit.yml' + - '!.github/workflows/build-docs.yml' + - '!.github/workflows/ci-nightly.yml' + - '!.github/workflows/ci-pixi-source-test.yml' + - '!.github/workflows/cleanup-pr-previews.yml' + - '!.github/workflows/coverage.yml' + - '!.github/workflows/pr-auto-label.yml' + - '!.github/workflows/pr-metadata-check.yml' + - '!.github/workflows/release-cuda-pathfinder.yml' + - '!.github/workflows/release-upload.yml' + - '!.github/workflows/release.yml' + - '!.github/workflows/security-suite.yml' + - '!.github/workflows/test-wheel-linux.yml' + - '!.github/workflows/test-wheel-windows.yml' + - '!.github/workflows/triagelabel.yml' + - '!ci/.ci-pipeline-regen.md' + - '!ci/ci-pipeline.svg' + - '!ci/cleanup-pr-previews' + - '!ci/test-matrix.yml' + - '!ci/tools/check_mempool_hygiene.py' + - '!ci/tools/check_pixi_cuda_version.py' + - '!ci/tools/check_release_notes.py' + - '!ci/tools/configure_driver_mode.ps1' + - '!ci/tools/download-wheels' + - '!ci/tools/guess_latest.sh' + - '!ci/tools/install_gpu_driver.ps1' + - '!ci/tools/install_gpu_driver.sh' + - '!ci/tools/run-tests' + - '!ci/tools/run_pytest_with_stack.py' + - '!ci/tools/setup-sanitizer' + - '!ci/tools/tests/**' + - '!ci/tools/validate-release-wheels' - name: Resolve reusable base artifacts id: baseline From eaed263c78ef65bf401a3607ec46e6cb734d0616 Mon Sep 17 00:00:00 2001 From: Keith Kraus Date: Fri, 14 Aug 2026 13:16:43 -0400 Subject: [PATCH 4/4] ci: compute selective workplan in Python --- .github/workflows/build-wheel.yml | 134 ++++---- .github/workflows/ci.yml | 389 ++++------------------- .github/workflows/test-sdist-linux.yml | 52 ++- .github/workflows/test-sdist-windows.yml | 46 ++- .github/workflows/test-wheel-linux.yml | 75 +++-- .github/workflows/test-wheel-windows.yml | 69 ++-- ci/tools/compute_ci_plan.py | 241 ++++++++++++++ ci/tools/tests/test_compute_ci_plan.py | 100 ++++++ 8 files changed, 567 insertions(+), 539 deletions(-) create mode 100644 ci/tools/compute_ci_plan.py create mode 100644 ci/tools/tests/test_compute_ci_plan.py diff --git a/.github/workflows/build-wheel.yml b/.github/workflows/build-wheel.yml index 390d6f88ae2..15e84819f1d 100644 --- a/.github/workflows/build-wheel.yml +++ b/.github/workflows/build-wheel.yml @@ -14,35 +14,8 @@ on: prev-cuda-version: required: true type: string - build-pathfinder: - required: false - type: boolean - default: true - build-bindings: - required: false - type: boolean - default: true - build-core: - required: false - type: boolean - default: true - build-python: - required: false - type: boolean - default: true - test-bindings: - required: false - type: boolean - default: true - test-core: - required: false - type: boolean - default: true - baseline-run-id: - required: false - type: string - default: "" - baseline-sha: + workplan: + description: JSON workplan. An empty value builds and tests everything. required: false type: string default: "" @@ -57,6 +30,15 @@ permissions: jobs: build: + env: + BUILD_PATHFINDER: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.pathfinder.needs_build }} + BUILD_BINDINGS: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.bindings.needs_build }} + BUILD_CORE: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.core.needs_build }} + BUILD_PYTHON: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.python.needs_build }} + TEST_BINDINGS: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.bindings.needs_test }} + TEST_CORE: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.core.needs_test }} + BASELINE_RUN_ID: ${{ inputs.workplan != '' && fromJSON(inputs.workplan).baseline.run_id || '' }} + BASELINE_SHA: ${{ inputs.workplan != '' && fromJSON(inputs.workplan).baseline.sha || '' }} strategy: fail-fast: false matrix: @@ -83,7 +65,7 @@ jobs: filter: blob:none - name: Install latest rapidsai/sccache - if: ${{ startsWith(inputs.host-platform, 'linux') && (inputs.build-bindings || inputs.build-core) }} + if: ${{ startsWith(inputs.host-platform, 'linux') && (env.BUILD_BINDINGS == 'true' || env.BUILD_CORE == 'true') }} run: | curl -fsSL "https://github.com/rapidsai/sccache/releases/latest/download/sccache-$(uname -m)-unknown-linux-musl.tar.gz" \ | sudo tar -C /usr/local/bin -xvzf - --wildcards --strip-components=1 -x '*/sccache' @@ -91,7 +73,7 @@ jobs: # xref: https://github.com/orgs/community/discussions/42856#discussioncomment-7678867 - name: Adding addtional GHA cache-related env vars - if: ${{ inputs.build-bindings || inputs.build-core }} + if: ${{ env.BUILD_BINDINGS == 'true' || env.BUILD_CORE == 'true' }} uses: actions/github-script@v9 with: script: | @@ -118,13 +100,13 @@ jobs: python-version: "3.12" - name: Set up MSVC - if: ${{ startsWith(inputs.host-platform, 'win') && (inputs.build-bindings || inputs.build-core || inputs.test-bindings || inputs.test-core) }} + if: ${{ startsWith(inputs.host-platform, 'win') && (env.BUILD_BINDINGS == 'true' || env.BUILD_CORE == 'true' || env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true') }} uses: step-security/msvc-dev-cmd@22c98154b708dbd743e6f27a933cf6ceba3305c4 # v1.13.1 - name: Set up yq # GitHub made an unprofessional decision to not provide it in their Windows VMs, # see https://github.com/actions/runner-images/issues/7443. - if: ${{ startsWith(inputs.host-platform, 'win') && inputs.build-core }} + if: ${{ startsWith(inputs.host-platform, 'win') && env.BUILD_CORE == 'true' }} env: YQ_VERSION: v4.52.5 YQ_SHA256: 47594981f3848a4b4447494adeca9555f908f7cf0a89c4da3fd0243a4631da1c @@ -162,20 +144,20 @@ jobs: # To keep the build workflow simple, all matrix jobs will build a wheel for later use within this workflow. - name: Build and check cuda.pathfinder wheel - if: ${{ inputs.build-pathfinder }} + if: ${{ env.BUILD_PATHFINDER == 'true' }} run: | pushd cuda_pathfinder pip wheel -v --no-deps . popd - name: Download reusable cuda.pathfinder wheel - if: ${{ !inputs.build-pathfinder }} + if: ${{ env.BUILD_PATHFINDER != 'true' }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: cuda-pathfinder-wheel path: cuda_pathfinder github-token: ${{ github.token }} - run-id: ${{ inputs.baseline-run-id }} + run-id: ${{ env.BASELINE_RUN_ID }} - name: List the cuda.pathfinder artifacts directory run: | @@ -190,12 +172,12 @@ jobs: # We only need/want a single pure python wheel, pick linux-64 index 0. # This is what we will use for testing & releasing. - name: Check cuda.pathfinder wheel - if: ${{ inputs.build-pathfinder && strategy.job-index == 0 && inputs.host-platform == 'linux-64' }} + if: ${{ env.BUILD_PATHFINDER == 'true' && strategy.job-index == 0 && inputs.host-platform == 'linux-64' }} run: | twine check --strict cuda_pathfinder/*.whl - name: Constrain builds to the local cuda.pathfinder wheel - if: ${{ inputs.build-bindings }} + if: ${{ env.BUILD_BINDINGS == 'true' }} run: | pathfinder_wheels=(cuda_pathfinder/cuda_pathfinder-*.whl) test "${#pathfinder_wheels[@]}" -eq 1 @@ -217,7 +199,7 @@ jobs: if-no-files-found: error - name: Set up mini CTK - if: ${{ inputs.build-bindings || inputs.build-core || inputs.test-bindings || inputs.test-core }} + if: ${{ env.BUILD_BINDINGS == 'true' || env.BUILD_CORE == 'true' || env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' }} uses: ./.github/actions/fetch_ctk continue-on-error: false with: @@ -225,7 +207,7 @@ jobs: cuda-version: ${{ inputs.cuda-version }} - name: Build cuda.bindings wheel - if: ${{ inputs.build-bindings }} + if: ${{ env.BUILD_BINDINGS == 'true' }} uses: pypa/cibuildwheel@4726cd35bb13f7bde50cf2761f2499ac7b3aa32c # v4.1.1 with: package-dir: ./cuda_bindings/ @@ -271,7 +253,7 @@ jobs: echo "ok!" - name: Report sccache stats (cuda.bindings) - if: ${{ inputs.build-bindings && inputs.host-platform != 'win-64' }} + if: ${{ env.BUILD_BINDINGS == 'true' && inputs.host-platform != 'win-64' }} uses: ./.github/actions/sccache-summary with: json-file: sccache_bindings.json @@ -279,13 +261,13 @@ jobs: build-step: "Build cuda.bindings wheel" - name: Download reusable cuda.bindings wheel - if: ${{ !inputs.build-bindings }} + if: ${{ env.BUILD_BINDINGS != 'true' }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: ${{ env.CUDA_BINDINGS_ARTIFACT_BASENAME }}-${{ inputs.baseline-sha }} + name: ${{ env.CUDA_BINDINGS_ARTIFACT_BASENAME }}-${{ env.BASELINE_SHA }} path: ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }} github-token: ${{ github.token }} - run-id: ${{ inputs.baseline-run-id }} + run-id: ${{ env.BASELINE_RUN_ID }} - name: List the cuda.bindings artifacts directory run: | @@ -298,12 +280,12 @@ jobs: ls -lahR ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }} - name: Check cuda.bindings wheel - if: ${{ inputs.build-bindings }} + if: ${{ env.BUILD_BINDINGS == 'true' }} run: | twine check --strict ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}/*.whl - name: Constrain cuda.core to the local cuda.bindings wheel - if: ${{ inputs.build-core }} + if: ${{ env.BUILD_CORE == 'true' }} run: | pathfinder_wheels=(cuda_pathfinder/cuda_pathfinder-*.whl) bindings_wheels=("${CUDA_BINDINGS_ARTIFACTS_DIR}"/cuda_bindings-"${BUILD_CUDA_MAJOR}".*.whl) @@ -332,7 +314,7 @@ jobs: if-no-files-found: error - name: Build cuda.core wheel - if: ${{ inputs.build-core }} + if: ${{ env.BUILD_CORE == 'true' }} uses: pypa/cibuildwheel@4726cd35bb13f7bde50cf2761f2499ac7b3aa32c # v4.1.1 with: package-dir: ./cuda_core/ @@ -380,7 +362,7 @@ jobs: echo "ok!" - name: Report sccache stats (cuda.core) - if: ${{ inputs.build-core && inputs.host-platform != 'win-64' }} + if: ${{ env.BUILD_CORE == 'true' && inputs.host-platform != 'win-64' }} uses: ./.github/actions/sccache-summary with: json-file: sccache_core.json @@ -388,7 +370,7 @@ jobs: build-step: "Build cuda.core wheel" - name: List the cuda.core artifacts directory and rename - if: ${{ inputs.build-core }} + if: ${{ env.BUILD_CORE == 'true' }} run: | if [[ "${{ inputs.host-platform }}" == win* ]]; then export CHOWN=chown @@ -411,17 +393,17 @@ jobs: ls -lahR ${{ env.CUDA_CORE_ARTIFACTS_DIR }} - name: Download reusable cuda.core wheel - if: ${{ !inputs.build-core }} + if: ${{ env.BUILD_CORE != 'true' }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: ${{ env.CUDA_CORE_ARTIFACT_BASENAME }}-${{ inputs.baseline-sha }} + name: ${{ env.CUDA_CORE_ARTIFACT_BASENAME }}-${{ env.BASELINE_SHA }} path: ${{ env.CUDA_CORE_ARTIFACTS_DIR }} github-token: ${{ github.token }} - run-id: ${{ inputs.baseline-run-id }} + run-id: ${{ env.BASELINE_RUN_ID }} # We only need/want a single pure python wheel, pick linux-64 index 0. - name: Build and check cuda-python wheel - if: ${{ inputs.build-python && strategy.job-index == 0 && inputs.host-platform == 'linux-64' }} + if: ${{ env.BUILD_PYTHON == 'true' && strategy.job-index == 0 && inputs.host-platform == 'linux-64' }} run: | pushd cuda_python pip wheel -v --no-deps . @@ -429,13 +411,13 @@ jobs: popd - name: Download reusable cuda-python wheel - if: ${{ !inputs.build-python && strategy.job-index == 0 && inputs.host-platform == 'linux-64' }} + if: ${{ env.BUILD_PYTHON != 'true' && strategy.job-index == 0 && inputs.host-platform == 'linux-64' }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: cuda-python-wheel path: cuda_python github-token: ${{ github.token }} - run-id: ${{ inputs.baseline-run-id }} + run-id: ${{ env.BASELINE_RUN_ID }} - name: List the cuda-python artifacts directory if: ${{ strategy.job-index == 0 && inputs.host-platform == 'linux-64' }} @@ -458,7 +440,7 @@ jobs: - name: Set up Python id: setup-python2 - if: ${{ inputs.test-bindings || inputs.test-core }} + if: ${{ env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' }} uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: ${{ matrix.python-version }} @@ -466,17 +448,17 @@ jobs: allow-prereleases: ${{ startsWith(matrix.python-version, '3.15') }} - name: Enable Scientific Python Nightly Wheels for Python 3.15 - if: ${{ (inputs.test-bindings || inputs.test-core) && startsWith(matrix.python-version, '3.15') }} + if: ${{ (env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true') && startsWith(matrix.python-version, '3.15') }} run: | echo "PIP_EXTRA_INDEX_URL=https://pypi.anaconda.org/scientific-python-nightly-wheels/simple" >> "$GITHUB_ENV" echo "PIP_ONLY_BINARY=numpy" >> "$GITHUB_ENV" - name: verify free-threaded build - if: ${{ (inputs.test-bindings || inputs.test-core) && endsWith(matrix.python-version, 't') }} + if: ${{ (env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true') && endsWith(matrix.python-version, 't') }} run: python -c 'import sys; assert not sys._is_gil_enabled()' - name: Set up Python include paths - if: ${{ inputs.test-bindings || inputs.test-core }} + if: ${{ env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' }} run: | if [[ "${{ inputs.host-platform }}" == linux* ]]; then echo "CPLUS_INCLUDE_PATH=${Python3_ROOT_DIR}/include/python${{ matrix.python-version }}" >> $GITHUB_ENV @@ -487,19 +469,19 @@ jobs: echo "PY_EXT_SUFFIX=$(python -c "import sysconfig; print(sysconfig.get_config_var('EXT_SUFFIX'))")" >> $GITHUB_ENV - name: Install cuda.pathfinder (required for next step) - if: ${{ inputs.test-bindings || inputs.test-core }} + if: ${{ env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' }} run: | pip install cuda_pathfinder/*.whl - name: Hide GNU link.exe so Meson finds MSVC link.exe - if: ${{ startsWith(inputs.host-platform, 'win') && (inputs.test-bindings || inputs.test-core) }} + if: ${{ startsWith(inputs.host-platform, 'win') && (env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true') }} run: | if [ -f "/c/Program Files/Git/usr/bin/link.exe" ]; then mv "/c/Program Files/Git/usr/bin/link.exe" "/c/Program Files/Git/usr/bin/link.exe.bak" fi - name: Build cuda.bindings Cython tests - if: ${{ inputs.test-bindings }} + if: ${{ env.TEST_BINDINGS == 'true' }} run: | pip install ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}/*.whl --group ./cuda_bindings/pyproject.toml:test pushd ${{ env.CUDA_BINDINGS_CYTHON_TESTS_DIR }} @@ -507,7 +489,7 @@ jobs: popd - name: Upload cuda.bindings Cython tests - if: ${{ inputs.test-bindings }} + if: ${{ env.TEST_BINDINGS == 'true' }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: ${{ env.CUDA_BINDINGS_ARTIFACT_NAME }}-tests @@ -515,10 +497,10 @@ jobs: if-no-files-found: error - name: Build cuda.core Cython tests - if: ${{ inputs.test-core }} + if: ${{ env.TEST_CORE == 'true' }} run: | pip install ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}/*.whl - if ${{ inputs.build-core }}; then + if ${{ env.BUILD_CORE == 'true' }}; then core_wheel=$(find "${{ env.CUDA_CORE_ARTIFACTS_DIR }}/cu${BUILD_CUDA_MAJOR}" -maxdepth 1 -type f -name '*.whl' -print -quit) else core_wheel=$(find "${{ env.CUDA_CORE_ARTIFACTS_DIR }}" -maxdepth 1 -type f -name '*.whl' -print -quit) @@ -533,7 +515,7 @@ jobs: popd - name: Upload cuda.core Cython tests - if: ${{ inputs.test-core }} + if: ${{ env.TEST_CORE == 'true' }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: ${{ env.CUDA_CORE_ARTIFACT_NAME }}-tests @@ -542,7 +524,7 @@ jobs: # Note: This overwrites CUDA_PATH etc - name: Set up mini CTK - if: ${{ inputs.build-core || inputs.test-core }} + if: ${{ env.BUILD_CORE == 'true' || env.TEST_CORE == 'true' }} uses: ./.github/actions/fetch_ctk continue-on-error: false with: @@ -551,13 +533,13 @@ jobs: cuda-path: "./cuda_toolkit_prev" - name: Build cuda.core test binaries - if: ${{ inputs.test-core }} + if: ${{ env.TEST_CORE == 'true' }} run: | nvcc --version python "${{ env.CUDA_CORE_TEST_BINARIES_DIR }}/build_test_binaries.py" - name: Upload cuda.core test binaries - if: ${{ inputs.test-core }} + if: ${{ env.TEST_CORE == 'true' }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: ${{ env.CUDA_CORE_ARTIFACT_NAME }}-test-binaries @@ -568,7 +550,7 @@ jobs: if-no-files-found: error - name: Download cuda.bindings build artifacts from the prior branch - if: ${{ inputs.build-core }} + if: ${{ env.BUILD_CORE == 'true' }} env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | @@ -597,7 +579,7 @@ jobs: rmdir $OLD_BASENAME - name: Constrain previous cuda.core to the downloaded cuda.bindings wheel - if: ${{ inputs.build-core }} + if: ${{ env.BUILD_CORE == 'true' }} run: | pathfinder_wheels=(cuda_pathfinder/cuda_pathfinder-*.whl) bindings_wheels=(cuda_bindings/dist-prev/cuda_bindings-"${BUILD_PREV_CUDA_MAJOR}".*.whl) @@ -619,7 +601,7 @@ jobs: } | tee wheel-constraints/cuda-core-prev.txt - name: Build cuda.core wheel - if: ${{ inputs.build-core }} + if: ${{ env.BUILD_CORE == 'true' }} uses: pypa/cibuildwheel@4726cd35bb13f7bde50cf2761f2499ac7b3aa32c # v4.1.1 with: package-dir: ./cuda_core/ @@ -667,7 +649,7 @@ jobs: echo "ok!" - name: Report sccache stats (cuda.core prev) - if: ${{ inputs.build-core && inputs.host-platform != 'win-64' }} + if: ${{ env.BUILD_CORE == 'true' && inputs.host-platform != 'win-64' }} uses: ./.github/actions/sccache-summary with: json-file: sccache_core_prev.json @@ -675,7 +657,7 @@ jobs: build-step: "Build cuda.core wheel" - name: List the cuda.core artifacts directory and rename - if: ${{ inputs.build-core }} + if: ${{ env.BUILD_CORE == 'true' }} run: | if [[ "${{ inputs.host-platform }}" == win* ]]; then export CHOWN=chown @@ -699,7 +681,7 @@ jobs: ls -lahR ${{ env.CUDA_CORE_ARTIFACTS_DIR }} - name: Merge cuda.core wheels - if: ${{ inputs.build-core }} + if: ${{ env.BUILD_CORE == 'true' }} run: | pip install wheel python ci/tools/merge_cuda_core_wheels.py \ @@ -708,7 +690,7 @@ jobs: --output-dir "${{ env.CUDA_CORE_ARTIFACTS_DIR }}" - name: Check cuda.core wheel - if: ${{ inputs.build-core }} + if: ${{ env.BUILD_CORE == 'true' }} run: | twine check --strict ${{ env.CUDA_CORE_ARTIFACTS_DIR }}/*.whl diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9120b2e062f..99a8904a1b2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -98,83 +98,7 @@ jobs: contents: read pull-requests: read outputs: - # Missing base artifacts or a skipped path filter fail open to the full pipeline. - core: >- - ${{ steps.baseline.outputs.available != 'true' || - steps.filter.outputs.changes == '' || - steps.filter.outputs.force_all == 'true' || - steps.filter.outputs.core_source == 'true' }} - build_pathfinder: >- - ${{ steps.baseline.outputs.available != 'true' || - steps.filter.outputs.changes == '' || - steps.filter.outputs.force_all == 'true' || - steps.filter.outputs.pathfinder_source == 'true' }} - build_bindings: >- - ${{ steps.baseline.outputs.available != 'true' || - steps.filter.outputs.changes == '' || - steps.filter.outputs.force_all == 'true' || - steps.filter.outputs.pathfinder_source == 'true' || - steps.filter.outputs.bindings_source == 'true' || - steps.filter.outputs.python_source == 'true' }} - build_core: >- - ${{ steps.baseline.outputs.available != 'true' || - steps.filter.outputs.changes == '' || - steps.filter.outputs.force_all == 'true' || - steps.filter.outputs.pathfinder_source == 'true' || - steps.filter.outputs.bindings_source == 'true' || - steps.filter.outputs.core_source == 'true' }} - build_python: >- - ${{ steps.baseline.outputs.available != 'true' || - steps.filter.outputs.changes == '' || - steps.filter.outputs.force_all == 'true' || - steps.filter.outputs.pathfinder_source == 'true' || - steps.filter.outputs.bindings_source == 'true' || - steps.filter.outputs.python_source == 'true' }} - test_pathfinder: >- - ${{ steps.baseline.outputs.available != 'true' || - steps.filter.outputs.changes == '' || - steps.filter.outputs.force_all == 'true' || - steps.filter.outputs.all_tests == 'true' || - steps.filter.outputs.pathfinder_source == 'true' || - steps.filter.outputs.pathfinder_tests == 'true' }} - test_bindings: >- - ${{ steps.baseline.outputs.available != 'true' || - steps.filter.outputs.changes == '' || - steps.filter.outputs.force_all == 'true' || - steps.filter.outputs.all_tests == 'true' || - steps.filter.outputs.pathfinder_source == 'true' || - steps.filter.outputs.bindings_source == 'true' || - steps.filter.outputs.bindings_tests == 'true' || - steps.filter.outputs.test_helpers == 'true' }} - test_core: >- - ${{ steps.baseline.outputs.available != 'true' || - steps.filter.outputs.changes == '' || - steps.filter.outputs.force_all == 'true' || - steps.filter.outputs.all_tests == 'true' || - steps.filter.outputs.pathfinder_source == 'true' || - steps.filter.outputs.bindings_source == 'true' || - steps.filter.outputs.core_source == 'true' || - steps.filter.outputs.core_tests == 'true' || - steps.filter.outputs.test_helpers == 'true' }} - pr_merge_base: ${{ steps.merge-base.outputs.sha }} - test_python: >- - ${{ steps.baseline.outputs.available != 'true' || - steps.filter.outputs.changes == '' || - steps.filter.outputs.force_all == 'true' || - steps.filter.outputs.all_tests == 'true' || - steps.filter.outputs.pathfinder_source == 'true' || - steps.filter.outputs.bindings_source == 'true' || - steps.filter.outputs.core_source == 'true' || - steps.filter.outputs.python_source == 'true' || - steps.filter.outputs.python_tests == 'true' }} - baseline_run_id: >- - ${{ steps.filter.outputs.changes != '' && - steps.baseline.outputs.available == 'true' && - steps.baseline.outputs.run_id || '' }} - baseline_sha: >- - ${{ steps.filter.outputs.changes != '' && - steps.baseline.outputs.available == 'true' && - steps.baseline.outputs.sha || '' }} + workplan: ${{ steps.workplan.outputs.workplan }} steps: - name: Checkout repository uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 @@ -216,166 +140,6 @@ jobs: base=$(git merge-base HEAD "origin/${BASE_REF}") echo "sha=${base}" >> "$GITHUB_OUTPUT" - - name: Classify changed paths - id: filter - if: ${{ startsWith(github.ref_name, 'pull-request/') }} - uses: dorny/paths-filter@ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d # v4.0.3 - with: - # The workflow runs on copy-pr-bot push branches, so compare the - # checked-out PR head against the base resolved from PR metadata. - base: ${{ steps.merge-base.outputs.sha }} - ref: ${{ github.sha }} - token: '' - predicate-quantifier: some-with-excludes - filters: | - pathfinder_source: - - 'cuda_pathfinder/**' - - '!cuda_pathfinder/docs/**' - - '!cuda_pathfinder/tests/**' - - '!cuda_pathfinder/examples/**' - - '!cuda_pathfinder/pixi.lock' - - '!cuda_pathfinder/pixi.toml' - - '!cuda_pathfinder/AGENTS.md' - - '!cuda_pathfinder/CLAUDE.md' - bindings_source: - - 'cuda_bindings/**' - - '!cuda_bindings/docs/**' - - '!cuda_bindings/tests/**' - - '!cuda_bindings/examples/**' - - '!cuda_bindings/pixi.lock' - - '!cuda_bindings/pixi.toml' - - '!cuda_bindings/AGENTS.md' - - '!cuda_bindings/CLAUDE.md' - core_source: - - 'cuda_core/**' - - '!cuda_core/docs/**' - - '!cuda_core/tests/**' - - '!cuda_core/examples/**' - - '!cuda_core/pixi.lock' - - '!cuda_core/pixi.toml' - - '!cuda_core/pytest.ini' - - '!cuda_core/AGENTS.md' - - '!cuda_core/CLAUDE.md' - python_source: - - 'cuda_python/**' - - '!cuda_python/docs/**' - - '!cuda_python/tests/**' - - '!cuda_python/examples/**' - - '!cuda_python/AGENTS.md' - - '!cuda_python/CLAUDE.md' - # cuda_python/README.md is a symlink to this packaging input. - - 'README.md' - pathfinder_tests: - - 'cuda_pathfinder/tests/**' - - 'cuda_pathfinder/examples/**' - - '!cuda_pathfinder/**/AGENTS.md' - - '!cuda_pathfinder/**/CLAUDE.md' - bindings_tests: - - 'cuda_bindings/tests/**' - - 'cuda_bindings/examples/**' - - 'benchmarks/cuda_bindings/**' - - '!benchmarks/cuda_bindings/pixi.lock' - - '!benchmarks/cuda_bindings/pixi.toml' - - '!cuda_bindings/**/AGENTS.md' - - '!cuda_bindings/**/CLAUDE.md' - core_tests: - - 'cuda_core/tests/**' - - 'cuda_core/examples/**' - - 'cuda_core/pytest.ini' - - '!cuda_core/**/AGENTS.md' - - '!cuda_core/**/CLAUDE.md' - python_tests: - - 'cuda_python/tests/**' - - 'cuda_python/examples/**' - - '!cuda_python/**/AGENTS.md' - - '!cuda_python/**/CLAUDE.md' - test_helpers: - - 'cuda_python_test_helpers/cuda_python_test_helpers/**' - # These files configure or implement wheel tests, but do not - # change any package artifact. - all_tests: - - '.github/workflows/test-wheel-linux.yml' - - '.github/workflows/test-wheel-windows.yml' - - 'ci/test-matrix.yml' - - 'ci/tools/configure_driver_mode.ps1' - - 'ci/tools/guess_latest.sh' - - 'ci/tools/install_gpu_driver.ps1' - - 'ci/tools/install_gpu_driver.sh' - - 'ci/tools/run-tests' - - 'ci/tools/setup-sanitizer' - # Shared infrastructure and unknown paths run the full pipeline. - # Exclude paths classified above and paths consumed only by an - # independent or unconditional CI job. - force_all: - - '**' - - '!cuda_pathfinder/**' - - '!cuda_bindings/**' - - '!cuda_core/**' - - '!cuda_python/**' - - '!cuda_python_test_helpers/**' - - '!benchmarks/cuda_bindings/**' - - '!benchmarks/cuda_core/**' - - '!.agents/**' - - '!.coveragerc' - - '!.gitignore' - - '!.pre-commit-config.yaml' - - '!.spdx-ignore' - - '!AGENTS.md' - - '!CLAUDE.md' - - '!CONTRIBUTING.md' - - '!LICENSE' - - '!README.md' - - '!SECURITY.md' - - '!context7.json' - - '!greptile.json' - - '!pixi.lock' - - '!pixi.toml' - - '!pytest.ini' - - '!ruff.toml' - - '!toolshed/**' - - '!.github/ISSUE_TEMPLATE/**' - - '!.github/PULL_REQUEST_TEMPLATE.md' - - '!.github/RELEASE-core.md' - - '!.github/actionlint.yaml' - - '!.github/actions/doc_preview/**' - - '!.github/actions/get_pr_number/**' - - '!.github/copy-pr-bot.yaml' - - '!.github/dependabot.yml' - - '!.github/labeler.yml' - - '!.github/workflows/backport.yml' - - '!.github/workflows/bandit.yml' - - '!.github/workflows/build-docs.yml' - - '!.github/workflows/ci-nightly.yml' - - '!.github/workflows/ci-pixi-source-test.yml' - - '!.github/workflows/cleanup-pr-previews.yml' - - '!.github/workflows/coverage.yml' - - '!.github/workflows/pr-auto-label.yml' - - '!.github/workflows/pr-metadata-check.yml' - - '!.github/workflows/release-cuda-pathfinder.yml' - - '!.github/workflows/release-upload.yml' - - '!.github/workflows/release.yml' - - '!.github/workflows/security-suite.yml' - - '!.github/workflows/test-wheel-linux.yml' - - '!.github/workflows/test-wheel-windows.yml' - - '!.github/workflows/triagelabel.yml' - - '!ci/.ci-pipeline-regen.md' - - '!ci/ci-pipeline.svg' - - '!ci/cleanup-pr-previews' - - '!ci/test-matrix.yml' - - '!ci/tools/check_mempool_hygiene.py' - - '!ci/tools/check_pixi_cuda_version.py' - - '!ci/tools/check_release_notes.py' - - '!ci/tools/configure_driver_mode.ps1' - - '!ci/tools/download-wheels' - - '!ci/tools/guess_latest.sh' - - '!ci/tools/install_gpu_driver.ps1' - - '!ci/tools/install_gpu_driver.sh' - - '!ci/tools/run-tests' - - '!ci/tools/run_pytest_with_stack.py' - - '!ci/tools/setup-sanitizer' - - '!ci/tools/tests/**' - - '!ci/tools/validate-release-wheels' - - name: Resolve reusable base artifacts id: baseline if: ${{ startsWith(github.ref_name, 'pull-request/') }} @@ -408,7 +172,7 @@ jobs: --workflow ci.yml \ --status success \ --limit 1 \ - --json databaseId,headSha,createdAt); then + --json databaseId,headSha); then unavailable fi @@ -469,11 +233,41 @@ jobs: echo "Reusable artifacts: run \`${run_id}\` at \`${baseline_sha}\` on \`${BASE_REF}\`." } >> "$GITHUB_STEP_SUMMARY" + - name: Test CI workplan planner + run: python3 -m unittest ci/tools/tests/test_compute_ci_plan.py + + - name: Compute CI workplan + id: workplan + env: + MERGE_BASE: ${{ steps.merge-base.outputs.sha }} + BASELINE_AVAILABLE: ${{ steps.baseline.outputs.available }} + BASELINE_RUN_ID: ${{ steps.baseline.outputs.run_id }} + BASELINE_SHA: ${{ steps.baseline.outputs.sha }} + run: | + set -euo pipefail + args=(--head "$GITHUB_SHA") + if [[ -n "$MERGE_BASE" ]]; then + args+=(--merge-base "$MERGE_BASE") + fi + if [[ "$BASELINE_AVAILABLE" == "true" ]]; then + args+=(--baseline-run-id "$BASELINE_RUN_ID" --baseline-sha "$BASELINE_SHA") + fi + + workplan=$(python3 ci/tools/compute_ci_plan.py "${args[@]}") + echo "workplan=$workplan" >> "$GITHUB_OUTPUT" + { + echo + echo "### CI workplan" + echo '```json' + jq . <<< "$workplan" + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + api-check-core-vs-release: name: API check (cuda_core vs. latest release) if: >- ${{ !fromJSON(needs.should-skip.outputs.skip) && - fromJSON(needs.detect-changes.outputs.core) }} + fromJSON(needs.detect-changes.outputs.workplan).jobs.core_api_checks }} runs-on: ubuntu-latest needs: - should-skip @@ -524,7 +318,7 @@ jobs: if: >- ${{ startsWith(github.ref_name, 'pull-request/') && !fromJSON(needs.should-skip.outputs.skip) && - fromJSON(needs.detect-changes.outputs.core) }} + fromJSON(needs.detect-changes.outputs.workplan).jobs.core_api_checks }} runs-on: ubuntu-latest needs: - should-skip @@ -542,7 +336,7 @@ jobs: shell: bash --noprofile --norc -euo pipefail {0} run: | git fetch --depth=1 --filter=blob:none origin \ - "${{ needs.detect-changes.outputs.pr_merge_base }}" + "${{ fromJSON(needs.detect-changes.outputs.workplan).merge_base }}" - name: Check cuda_core public API id: griffe @@ -550,7 +344,7 @@ jobs: with: package-name: cuda.core package-dir: cuda_core - merge-base: ${{ needs.detect-changes.outputs.pr_merge_base }} + merge-base: ${{ fromJSON(needs.detect-changes.outputs.workplan).merge_base }} # NOTE: Build jobs are intentionally split by platform rather than using a single # matrix. This lets each test job consume its platform-specific artifacts as @@ -580,14 +374,7 @@ jobs: host-platform: ${{ matrix.host-platform }} cuda-version: ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} prev-cuda-version: ${{ needs.ci-vars.outputs.CUDA_PREV_BUILD_VER }} - build-pathfinder: ${{ fromJSON(needs.detect-changes.outputs.build_pathfinder) }} - build-bindings: ${{ fromJSON(needs.detect-changes.outputs.build_bindings) }} - build-core: ${{ fromJSON(needs.detect-changes.outputs.build_core) }} - build-python: ${{ fromJSON(needs.detect-changes.outputs.build_python) }} - test-bindings: ${{ fromJSON(needs.detect-changes.outputs.test_bindings) }} - test-core: ${{ fromJSON(needs.detect-changes.outputs.test_core) }} - baseline-run-id: ${{ needs.detect-changes.outputs.baseline_run_id }} - baseline-sha: ${{ needs.detect-changes.outputs.baseline_sha }} + workplan: ${{ needs.detect-changes.outputs.workplan }} # See build-linux-64 for why build jobs are split by platform. build-linux-aarch64: @@ -604,14 +391,7 @@ jobs: if: ${{ github.repository_owner == 'nvidia' && !fromJSON(needs.should-skip.outputs.skip) && !fromJSON(needs.should-skip.outputs.doc-only) && - (fromJSON(needs.detect-changes.outputs.build_pathfinder) || - fromJSON(needs.detect-changes.outputs.build_bindings) || - fromJSON(needs.detect-changes.outputs.build_core) || - fromJSON(needs.detect-changes.outputs.build_python) || - fromJSON(needs.detect-changes.outputs.test_pathfinder) || - fromJSON(needs.detect-changes.outputs.test_bindings) || - fromJSON(needs.detect-changes.outputs.test_core) || - fromJSON(needs.detect-changes.outputs.test_python)) }} + fromJSON(needs.detect-changes.outputs.workplan).jobs.platform_builds }} permissions: actions: read contents: read @@ -621,14 +401,7 @@ jobs: host-platform: ${{ matrix.host-platform }} cuda-version: ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} prev-cuda-version: ${{ needs.ci-vars.outputs.CUDA_PREV_BUILD_VER }} - build-pathfinder: ${{ fromJSON(needs.detect-changes.outputs.build_pathfinder) }} - build-bindings: ${{ fromJSON(needs.detect-changes.outputs.build_bindings) }} - build-core: ${{ fromJSON(needs.detect-changes.outputs.build_core) }} - build-python: ${{ fromJSON(needs.detect-changes.outputs.build_python) }} - test-bindings: ${{ fromJSON(needs.detect-changes.outputs.test_bindings) }} - test-core: ${{ fromJSON(needs.detect-changes.outputs.test_core) }} - baseline-run-id: ${{ needs.detect-changes.outputs.baseline_run_id }} - baseline-sha: ${{ needs.detect-changes.outputs.baseline_sha }} + workplan: ${{ needs.detect-changes.outputs.workplan }} # See build-linux-64 for why build jobs are split by platform. build-windows: @@ -645,14 +418,7 @@ jobs: if: ${{ github.repository_owner == 'nvidia' && !fromJSON(needs.should-skip.outputs.skip) && !fromJSON(needs.should-skip.outputs.doc-only) && - (fromJSON(needs.detect-changes.outputs.build_pathfinder) || - fromJSON(needs.detect-changes.outputs.build_bindings) || - fromJSON(needs.detect-changes.outputs.build_core) || - fromJSON(needs.detect-changes.outputs.build_python) || - fromJSON(needs.detect-changes.outputs.test_pathfinder) || - fromJSON(needs.detect-changes.outputs.test_bindings) || - fromJSON(needs.detect-changes.outputs.test_core) || - fromJSON(needs.detect-changes.outputs.test_python)) }} + fromJSON(needs.detect-changes.outputs.workplan).jobs.platform_builds }} permissions: actions: read contents: read @@ -662,14 +428,7 @@ jobs: host-platform: ${{ matrix.host-platform }} cuda-version: ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} prev-cuda-version: ${{ needs.ci-vars.outputs.CUDA_PREV_BUILD_VER }} - build-pathfinder: ${{ fromJSON(needs.detect-changes.outputs.build_pathfinder) }} - build-bindings: ${{ fromJSON(needs.detect-changes.outputs.build_bindings) }} - build-core: ${{ fromJSON(needs.detect-changes.outputs.build_core) }} - build-python: ${{ fromJSON(needs.detect-changes.outputs.build_python) }} - test-bindings: ${{ fromJSON(needs.detect-changes.outputs.test_bindings) }} - test-core: ${{ fromJSON(needs.detect-changes.outputs.test_core) }} - baseline-run-id: ${{ needs.detect-changes.outputs.baseline_run_id }} - baseline-sha: ${{ needs.detect-changes.outputs.baseline_sha }} + workplan: ${{ needs.detect-changes.outputs.workplan }} # NOTE: test-sdist jobs are split by platform (mirroring build-* and test-wheel-*) # so platform-specific sources (e.g. cuda_bindings/*_windows.pyx selected by @@ -687,10 +446,7 @@ jobs: if: ${{ github.repository_owner == 'nvidia' && !fromJSON(needs.should-skip.outputs.skip) && !fromJSON(needs.should-skip.outputs.doc-only) && - (fromJSON(needs.detect-changes.outputs.build_pathfinder) || - fromJSON(needs.detect-changes.outputs.build_bindings) || - fromJSON(needs.detect-changes.outputs.build_core) || - fromJSON(needs.detect-changes.outputs.build_python)) }} + fromJSON(needs.detect-changes.outputs.workplan).jobs.sdist_tests }} permissions: actions: read contents: read @@ -699,10 +455,7 @@ jobs: with: host-platform: linux-64 cuda-version: ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} - build-pathfinder: ${{ fromJSON(needs.detect-changes.outputs.build_pathfinder) }} - build-bindings: ${{ fromJSON(needs.detect-changes.outputs.build_bindings) }} - build-core: ${{ fromJSON(needs.detect-changes.outputs.build_core) }} - build-python: ${{ fromJSON(needs.detect-changes.outputs.build_python) }} + workplan: ${{ needs.detect-changes.outputs.workplan }} # See test-sdist-linux for why sdist test jobs are split by platform. test-sdist-windows: @@ -716,10 +469,7 @@ jobs: if: ${{ github.repository_owner == 'nvidia' && !fromJSON(needs.should-skip.outputs.skip) && !fromJSON(needs.should-skip.outputs.doc-only) && - (fromJSON(needs.detect-changes.outputs.build_pathfinder) || - fromJSON(needs.detect-changes.outputs.build_bindings) || - fromJSON(needs.detect-changes.outputs.build_core) || - fromJSON(needs.detect-changes.outputs.build_python)) }} + fromJSON(needs.detect-changes.outputs.workplan).jobs.sdist_tests }} permissions: actions: read contents: read @@ -728,10 +478,7 @@ jobs: with: host-platform: win-64 cuda-version: ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} - build-pathfinder: ${{ fromJSON(needs.detect-changes.outputs.build_pathfinder) }} - build-bindings: ${{ fromJSON(needs.detect-changes.outputs.build_bindings) }} - build-core: ${{ fromJSON(needs.detect-changes.outputs.build_core) }} - build-python: ${{ fromJSON(needs.detect-changes.outputs.build_python) }} + workplan: ${{ needs.detect-changes.outputs.workplan }} # NOTE: Test jobs are split by platform for the same reason as build jobs (see # build-linux-64). Keep these job definitions textually identical except for: @@ -747,10 +494,7 @@ jobs: name: Test ${{ matrix.host-platform }} if: ${{ github.repository_owner == 'nvidia' && !fromJSON(needs.should-skip.outputs.doc-only) && - (fromJSON(needs.detect-changes.outputs.test_pathfinder) || - fromJSON(needs.detect-changes.outputs.test_bindings) || - fromJSON(needs.detect-changes.outputs.test_core) || - fromJSON(needs.detect-changes.outputs.test_python)) }} + fromJSON(needs.detect-changes.outputs.workplan).jobs.wheel_tests }} permissions: actions: read contents: read # This is required for actions/checkout @@ -766,10 +510,7 @@ jobs: host-platform: ${{ matrix.host-platform }} build-ctk-ver: ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} nruns: ${{ (github.event_name == 'schedule' && 5) || 1}} - test-pathfinder: ${{ fromJSON(needs.detect-changes.outputs.test_pathfinder) }} - test-bindings: ${{ fromJSON(needs.detect-changes.outputs.test_bindings) }} - test-core: ${{ fromJSON(needs.detect-changes.outputs.test_core) }} - test-python: ${{ fromJSON(needs.detect-changes.outputs.test_python) }} + workplan: ${{ needs.detect-changes.outputs.workplan }} # See test-linux-64 for why test jobs are split by platform. test-linux-aarch64: @@ -781,10 +522,7 @@ jobs: name: Test ${{ matrix.host-platform }} if: ${{ github.repository_owner == 'nvidia' && !fromJSON(needs.should-skip.outputs.doc-only) && - (fromJSON(needs.detect-changes.outputs.test_pathfinder) || - fromJSON(needs.detect-changes.outputs.test_bindings) || - fromJSON(needs.detect-changes.outputs.test_core) || - fromJSON(needs.detect-changes.outputs.test_python)) }} + fromJSON(needs.detect-changes.outputs.workplan).jobs.wheel_tests }} permissions: actions: read contents: read # This is required for actions/checkout @@ -801,10 +539,7 @@ jobs: host-platform: ${{ matrix.host-platform }} build-ctk-ver: ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} nruns: ${{ (github.event_name == 'schedule' && 5) || 1}} - test-pathfinder: ${{ fromJSON(needs.detect-changes.outputs.test_pathfinder) }} - test-bindings: ${{ fromJSON(needs.detect-changes.outputs.test_bindings) }} - test-core: ${{ fromJSON(needs.detect-changes.outputs.test_core) }} - test-python: ${{ fromJSON(needs.detect-changes.outputs.test_python) }} + workplan: ${{ needs.detect-changes.outputs.workplan }} # See test-linux-64 for why test jobs are split by platform. test-windows: @@ -816,10 +551,7 @@ jobs: name: Test ${{ matrix.host-platform }} if: ${{ github.repository_owner == 'nvidia' && !fromJSON(needs.should-skip.outputs.doc-only) && - (fromJSON(needs.detect-changes.outputs.test_pathfinder) || - fromJSON(needs.detect-changes.outputs.test_bindings) || - fromJSON(needs.detect-changes.outputs.test_core) || - fromJSON(needs.detect-changes.outputs.test_python)) }} + fromJSON(needs.detect-changes.outputs.workplan).jobs.wheel_tests }} permissions: actions: read contents: read # This is required for actions/checkout @@ -836,10 +568,7 @@ jobs: host-platform: ${{ matrix.host-platform }} build-ctk-ver: ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} nruns: ${{ (github.event_name == 'schedule' && 5) || 1}} - test-pathfinder: ${{ fromJSON(needs.detect-changes.outputs.test_pathfinder) }} - test-bindings: ${{ fromJSON(needs.detect-changes.outputs.test_bindings) }} - test-core: ${{ fromJSON(needs.detect-changes.outputs.test_core) }} - test-python: ${{ fromJSON(needs.detect-changes.outputs.test_python) }} + workplan: ${{ needs.detect-changes.outputs.workplan }} doc: name: Docs @@ -927,15 +656,10 @@ jobs: fi doc_only="${{ needs.should-skip.outputs.doc-only }}" - build_selected="${{ needs.detect-changes.outputs.build_pathfinder == 'true' || - needs.detect-changes.outputs.build_bindings == 'true' || - needs.detect-changes.outputs.build_core == 'true' || - needs.detect-changes.outputs.build_python == 'true' }}" - test_selected="${{ needs.detect-changes.outputs.test_pathfinder == 'true' || - needs.detect-changes.outputs.test_bindings == 'true' || - needs.detect-changes.outputs.test_core == 'true' || - needs.detect-changes.outputs.test_python == 'true' }}" - run_core_api_check="${{ needs.detect-changes.outputs.core == 'true' }}" + platform_selected="${{ needs.detect-changes.outputs.workplan && fromJSON(needs.detect-changes.outputs.workplan).jobs.platform_builds || false }}" + build_selected="${{ needs.detect-changes.outputs.workplan && fromJSON(needs.detect-changes.outputs.workplan).jobs.sdist_tests || false }}" + test_selected="${{ needs.detect-changes.outputs.workplan && fromJSON(needs.detect-changes.outputs.workplan).jobs.wheel_tests || false }}" + run_core_api_check="${{ needs.detect-changes.outputs.workplan && fromJSON(needs.detect-changes.outputs.workplan).jobs.core_api_checks || false }}" is_pr="${{ startsWith(github.ref_name, 'pull-request/') }}" status="success" check_result() { @@ -958,8 +682,7 @@ jobs: # Platform builds run whenever any package needs to be built or tested. expected="skipped" - if [[ "$doc_only" != "true" && - ( "$build_selected" == "true" || "$test_selected" == "true" ) ]]; then + if [[ "$doc_only" != "true" && "$platform_selected" == "true" ]]; then expected="success" fi check_result "build-linux-aarch64" "$expected" "${{ needs.build-linux-aarch64.result }}" diff --git a/.github/workflows/test-sdist-linux.yml b/.github/workflows/test-sdist-linux.yml index f0f64492f2d..ba7cdfc6ef1 100644 --- a/.github/workflows/test-sdist-linux.yml +++ b/.github/workflows/test-sdist-linux.yml @@ -11,22 +11,11 @@ on: cuda-version: required: true type: string - build-pathfinder: + workplan: + description: JSON workplan. An empty value builds everything. required: false - default: true - type: boolean - build-bindings: - required: false - default: true - type: boolean - build-core: - required: false - default: true - type: boolean - build-python: - required: false - default: true - type: boolean + default: "" + type: string defaults: run: @@ -39,7 +28,12 @@ permissions: jobs: test-sdist: name: Test sdist builds - if: ${{ inputs.build-pathfinder || inputs.build-bindings || inputs.build-core || inputs.build-python }} + if: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).jobs.sdist_tests }} + env: + BUILD_PATHFINDER: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.pathfinder.needs_build }} + BUILD_BINDINGS: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.bindings.needs_build }} + BUILD_CORE: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.core.needs_build }} + BUILD_PYTHON: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.python.needs_build }} timeout-minutes: 60 runs-on: linux-amd64-cpu8 steps: @@ -61,26 +55,26 @@ jobs: # Pure Python packages -- no CTK needed. - name: Build cuda.pathfinder sdist and wheel-from-sdist - if: ${{ inputs.build-pathfinder }} + if: ${{ env.BUILD_PATHFINDER == 'true' }} run: | python -m build --sdist cuda_pathfinder/ pip wheel --no-deps --wheel-dir cuda_pathfinder/dist cuda_pathfinder/dist/*.tar.gz - name: Build cuda-python sdist and wheel-from-sdist - if: ${{ inputs.build-python }} + if: ${{ env.BUILD_PYTHON == 'true' }} run: | python -m build --sdist cuda_python/ pip wheel --no-deps --wheel-dir cuda_python/dist cuda_python/dist/*.tar.gz - name: Download cuda.pathfinder wheel - if: ${{ !inputs.build-pathfinder && (inputs.build-bindings || inputs.build-core) }} + if: ${{ env.BUILD_PATHFINDER != 'true' && (env.BUILD_BINDINGS == 'true' || env.BUILD_CORE == 'true') }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: cuda-pathfinder-wheel path: cuda_pathfinder/dist - name: Constrain builds to the local cuda.pathfinder wheel - if: ${{ inputs.build-bindings }} + if: ${{ env.BUILD_BINDINGS == 'true' }} run: | pathfinder_wheels=(cuda_pathfinder/dist/cuda_pathfinder-*.whl) test "${#pathfinder_wheels[@]}" -eq 1 @@ -93,14 +87,14 @@ jobs: # The env vars ACTIONS_CACHE_SERVICE_V2, ACTIONS_RESULTS_URL, and ACTIONS_RUNTIME_TOKEN # are exposed by this action. - name: Enable sccache - if: ${{ inputs.build-bindings || inputs.build-core }} + if: ${{ env.BUILD_BINDINGS == 'true' || env.BUILD_CORE == 'true' }} uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # 0.0.10 with: disable_annotations: 'true' # xref: https://github.com/orgs/community/discussions/42856#discussioncomment-7678867 - name: Adding additional GHA cache-related env vars - if: ${{ inputs.build-bindings || inputs.build-core }} + if: ${{ env.BUILD_BINDINGS == 'true' || env.BUILD_CORE == 'true' }} uses: actions/github-script@v9 with: script: | @@ -108,14 +102,14 @@ jobs: core.exportVariable('ACTIONS_RUNTIME_URL', process.env['ACTIONS_RUNTIME_URL']) - name: Setup proxy cache - if: ${{ inputs.build-bindings || inputs.build-core }} + if: ${{ env.BUILD_BINDINGS == 'true' || env.BUILD_CORE == 'true' }} uses: nv-gha-runners/setup-proxy-cache@main continue-on-error: true with: enable-apt: true - name: Set up mini CTK - if: ${{ inputs.build-bindings || inputs.build-core }} + if: ${{ env.BUILD_BINDINGS == 'true' || env.BUILD_CORE == 'true' }} uses: ./.github/actions/fetch_ctk continue-on-error: false with: @@ -125,7 +119,7 @@ jobs: # cuda_bindings/setup.py parses CUDA headers at import time, so CUDA_PATH # (set by fetch_ctk) must be available for both sdist and wheel builds. - name: Build cuda.bindings sdist and wheel-from-sdist - if: ${{ inputs.build-bindings }} + if: ${{ env.BUILD_BINDINGS == 'true' }} run: | export CUDA_PYTHON_PARALLEL_LEVEL=$(nproc) export CC="sccache cc" @@ -136,14 +130,14 @@ jobs: pip wheel --no-deps --wheel-dir cuda_bindings/dist cuda_bindings/dist/*.tar.gz - name: Download cuda.bindings wheel - if: ${{ !inputs.build-bindings && inputs.build-core }} + if: ${{ env.BUILD_BINDINGS != 'true' && env.BUILD_CORE == 'true' }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: cuda-bindings-python312-cuda${{ inputs.cuda-version }}-${{ inputs.host-platform }}-${{ github.sha }} path: cuda_bindings/dist - name: Constrain cuda.core to the local cuda.bindings wheel - if: ${{ inputs.build-core }} + if: ${{ env.BUILD_CORE == 'true' }} run: | CUDA_MAJOR="$(echo '${{ inputs.cuda-version }}' | cut -d. -f1)" pathfinder_wheels=(cuda_pathfinder/dist/cuda_pathfinder-*.whl) @@ -164,7 +158,7 @@ jobs: # wheel-from-sdist needs CTK and cuda-bindings (dynamic build dep via # get_requires_for_build_wheel in build_hooks.py). - name: Build cuda.core sdist and wheel-from-sdist - if: ${{ inputs.build-core }} + if: ${{ env.BUILD_CORE == 'true' }} run: | export CUDA_PYTHON_PARALLEL_LEVEL=$(nproc) export CUDA_CORE_BUILD_MAJOR="$(echo '${{ inputs.cuda-version }}' | cut -d. -f1)" @@ -176,5 +170,5 @@ jobs: pip wheel --no-deps --wheel-dir cuda_core/dist cuda_core/dist/*.tar.gz - name: Show sccache stats - if: ${{ always() && (inputs.build-bindings || inputs.build-core) }} + if: ${{ always() && (env.BUILD_BINDINGS == 'true' || env.BUILD_CORE == 'true') }} run: sccache --show-stats diff --git a/.github/workflows/test-sdist-windows.yml b/.github/workflows/test-sdist-windows.yml index 5451d20429e..a0594800eba 100644 --- a/.github/workflows/test-sdist-windows.yml +++ b/.github/workflows/test-sdist-windows.yml @@ -17,22 +17,11 @@ on: cuda-version: required: true type: string - build-pathfinder: + workplan: + description: JSON workplan. An empty value builds everything. required: false - default: true - type: boolean - build-bindings: - required: false - default: true - type: boolean - build-core: - required: false - default: true - type: boolean - build-python: - required: false - default: true - type: boolean + default: "" + type: string defaults: run: @@ -45,7 +34,12 @@ permissions: jobs: test-sdist: name: Test sdist builds - if: ${{ inputs.build-pathfinder || inputs.build-bindings || inputs.build-core || inputs.build-python }} + if: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).jobs.sdist_tests }} + env: + BUILD_PATHFINDER: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.pathfinder.needs_build }} + BUILD_BINDINGS: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.bindings.needs_build }} + BUILD_CORE: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.core.needs_build }} + BUILD_PYTHON: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.python.needs_build }} timeout-minutes: 60 runs-on: windows-2022 steps: @@ -63,7 +57,7 @@ jobs: python-version: "3.12" - name: Set up MSVC - if: ${{ inputs.build-bindings || inputs.build-core }} + if: ${{ env.BUILD_BINDINGS == 'true' || env.BUILD_CORE == 'true' }} uses: step-security/msvc-dev-cmd@22c98154b708dbd743e6f27a933cf6ceba3305c4 # v1.13.1 - name: Install build tools @@ -71,26 +65,26 @@ jobs: # Pure Python packages -- no CTK needed. - name: Build cuda.pathfinder sdist and wheel-from-sdist - if: ${{ inputs.build-pathfinder }} + if: ${{ env.BUILD_PATHFINDER == 'true' }} run: | python -m build --sdist cuda_pathfinder/ pip wheel --no-deps --wheel-dir cuda_pathfinder/dist cuda_pathfinder/dist/*.tar.gz - name: Build cuda-python sdist and wheel-from-sdist - if: ${{ inputs.build-python }} + if: ${{ env.BUILD_PYTHON == 'true' }} run: | python -m build --sdist cuda_python/ pip wheel --no-deps --wheel-dir cuda_python/dist cuda_python/dist/*.tar.gz - name: Download cuda.pathfinder wheel - if: ${{ !inputs.build-pathfinder && (inputs.build-bindings || inputs.build-core) }} + if: ${{ env.BUILD_PATHFINDER != 'true' && (env.BUILD_BINDINGS == 'true' || env.BUILD_CORE == 'true') }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: cuda-pathfinder-wheel path: cuda_pathfinder/dist - name: Constrain builds to the local cuda.pathfinder wheel - if: ${{ inputs.build-bindings }} + if: ${{ env.BUILD_BINDINGS == 'true' }} run: | pathfinder_wheels=(cuda_pathfinder/dist/cuda_pathfinder-*.whl) test "${#pathfinder_wheels[@]}" -eq 1 @@ -103,7 +97,7 @@ jobs: # smoke test, not a production build; see build-wheel.yml which also # limits sccache to Linux). - name: Set up mini CTK - if: ${{ inputs.build-bindings || inputs.build-core }} + if: ${{ env.BUILD_BINDINGS == 'true' || env.BUILD_CORE == 'true' }} uses: ./.github/actions/fetch_ctk continue-on-error: false with: @@ -115,7 +109,7 @@ jobs: # Constraint paths are passed as native Windows paths because the pip # subprocesses run outside Git Bash. - name: Build cuda.bindings sdist and wheel-from-sdist - if: ${{ inputs.build-bindings }} + if: ${{ env.BUILD_BINDINGS == 'true' }} run: | export CUDA_PYTHON_PARALLEL_LEVEL=$(nproc) export PIP_BUILD_CONSTRAINT="$(cygpath -w "$(pwd)/wheel-constraints/cuda-bindings.txt")" @@ -124,14 +118,14 @@ jobs: pip wheel --no-deps --wheel-dir cuda_bindings/dist cuda_bindings/dist/*.tar.gz - name: Download cuda.bindings wheel - if: ${{ !inputs.build-bindings && inputs.build-core }} + if: ${{ env.BUILD_BINDINGS != 'true' && env.BUILD_CORE == 'true' }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: cuda-bindings-python312-cuda${{ inputs.cuda-version }}-${{ inputs.host-platform }}-${{ github.sha }} path: cuda_bindings/dist - name: Constrain cuda.core to the local cuda.bindings wheel - if: ${{ inputs.build-core }} + if: ${{ env.BUILD_CORE == 'true' }} run: | CUDA_MAJOR="$(echo '${{ inputs.cuda-version }}' | cut -d. -f1)" pathfinder_wheels=(cuda_pathfinder/dist/cuda_pathfinder-*.whl) @@ -152,7 +146,7 @@ jobs: # wheel-from-sdist needs CTK and cuda-bindings (dynamic build dep via # get_requires_for_build_wheel in build_hooks.py). - name: Build cuda.core sdist and wheel-from-sdist - if: ${{ inputs.build-core }} + if: ${{ env.BUILD_CORE == 'true' }} run: | export CUDA_PYTHON_PARALLEL_LEVEL=$(nproc) export CUDA_CORE_BUILD_MAJOR="$(echo '${{ inputs.cuda-version }}' | cut -d. -f1)" diff --git a/.github/workflows/test-wheel-linux.yml b/.github/workflows/test-wheel-linux.yml index 8134a6844fd..4b7f2d64f8e 100644 --- a/.github/workflows/test-wheel-linux.yml +++ b/.github/workflows/test-wheel-linux.yml @@ -22,18 +22,10 @@ on: nruns: type: number default: 1 - test-pathfinder: - type: boolean - default: true - test-bindings: - type: boolean - default: true - test-core: - type: boolean - default: true - test-python: - type: boolean - default: true + workplan: + description: JSON workplan. An empty value tests everything. + type: string + default: "" run-id: description: > Workflow run ID to download artifacts from. @@ -106,6 +98,11 @@ jobs: echo "OLD_BRANCH=${OLD_BRANCH}" >> "$GITHUB_OUTPUT" test: + env: + TEST_PATHFINDER: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.pathfinder.needs_test }} + TEST_BINDINGS: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.bindings.needs_test }} + TEST_CORE: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.core.needs_test }} + TEST_PYTHON: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.python.needs_test }} name: Python ${{ matrix.PY_VER }}, CUDA ${{ matrix.CUDA_VER }} (${{ (matrix.LOCAL_CTK == '1' && 'local') || 'wheels' }}), GPU ${{ matrix.GPU }}${{ matrix.GPU_COUNT != '1' && format(' (x{0})', matrix.GPU_COUNT) || '' }}${{ matrix.FLAVOR && format(', {0}', matrix.FLAVOR) || '' }}${{ matrix.ENV.TORCH_VER && format(', {0}+{1}', matrix.ENV.TORCH_VER, matrix.ENV.TORCH_CUDA) || '' }}${{ matrix.ENV.MODE == 'nightly-numba-cuda' && ', latest' || '' }} timeout-minutes: 60 needs: compute-matrix @@ -164,7 +161,7 @@ jobs: LOCAL_CTK: ${{ matrix.LOCAL_CTK }} PY_VER: ${{ matrix.PY_VER }} SHA: ${{ inputs.sha || github.sha }} - SKIP_BINDINGS_TEST_OVERRIDE: ${{ !inputs.test-bindings && '1' || '0' }} + SKIP_BINDINGS_TEST_OVERRIDE: ${{ env.TEST_BINDINGS != 'true' && '1' || '0' }} run: ./ci/tools/env-vars test - name: Apply extra matrix environment variables @@ -174,7 +171,7 @@ jobs: run: echo "$MATRIX_ENV" | jq -r 'to_entries[] | "\(.key)=\(.value)"' >> "$GITHUB_ENV" - name: Download cuda-pathfinder build artifacts - if: ${{ inputs.test-pathfinder || inputs.test-bindings || inputs.test-core || inputs.test-python }} + if: ${{ env.TEST_PATHFINDER == 'true' || env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' || env.TEST_PYTHON == 'true' }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: cuda-pathfinder-wheel @@ -183,7 +180,7 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} - name: Download cuda-python build artifacts - if: ${{ inputs.test-python && env.BINDINGS_SOURCE == 'main' }} + if: ${{ env.TEST_PYTHON == 'true' && env.BINDINGS_SOURCE == 'main' }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: cuda-python-wheel @@ -192,7 +189,7 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} - name: Download cuda.bindings build artifacts - if: ${{ (inputs.test-bindings || inputs.test-core || inputs.test-python) && + if: ${{ (env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' || env.TEST_PYTHON == 'true') && env.BINDINGS_SOURCE == 'main' }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: @@ -202,7 +199,7 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} - name: Download cuda-python & cuda.bindings build artifacts from the prior branch - if: ${{ (inputs.test-bindings || inputs.test-core || inputs.test-python) && + if: ${{ (env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' || env.TEST_PYTHON == 'true') && env.BINDINGS_SOURCE == 'backport' }} env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -228,7 +225,7 @@ jobs: mv $OLD_BASENAME/*.whl "${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}"/ rmdir $OLD_BASENAME - if ${{ inputs.test-python }}; then + if ${{ env.TEST_PYTHON == 'true' }}; then gh run download $LATEST_PRIOR_RUN_ID -p cuda-python-wheel -R NVIDIA/cuda-python ls -al cuda-python-wheel mv cuda-python-wheel/*.whl . @@ -236,20 +233,20 @@ jobs: fi - name: Display structure of downloaded cuda-python artifacts - if: ${{ inputs.test-python && env.BINDINGS_SOURCE != 'published' }} + if: ${{ env.TEST_PYTHON == 'true' && env.BINDINGS_SOURCE != 'published' }} run: | pwd ls -lah cuda_python*.whl cuda_pathfinder/ - name: Display structure of downloaded cuda.bindings artifacts - if: ${{ (inputs.test-bindings || inputs.test-core || inputs.test-python) && + if: ${{ (env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' || env.TEST_PYTHON == 'true') && env.BINDINGS_SOURCE != 'published' }} run: | pwd ls -lahR $CUDA_BINDINGS_ARTIFACTS_DIR - name: Download cuda.bindings Cython tests - if: ${{ inputs.test-bindings && env.SKIP_CYTHON_TEST == '0' }} + if: ${{ env.TEST_BINDINGS == 'true' && env.SKIP_CYTHON_TEST == '0' }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: ${{ env.CUDA_BINDINGS_ARTIFACT_NAME }}-tests @@ -258,13 +255,13 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} - name: Display structure of downloaded cuda.bindings Cython tests - if: ${{ inputs.test-bindings && env.SKIP_CYTHON_TEST == '0' }} + if: ${{ env.TEST_BINDINGS == 'true' && env.SKIP_CYTHON_TEST == '0' }} run: | pwd ls -lahR $CUDA_BINDINGS_CYTHON_TESTS_DIR - name: Download cuda.core build artifacts - if: ${{ inputs.test-core }} + if: ${{ env.TEST_CORE == 'true' }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: ${{ env.CUDA_CORE_ARTIFACT_NAME }} @@ -273,13 +270,13 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} - name: Display structure of downloaded cuda.core build artifacts - if: ${{ inputs.test-core }} + if: ${{ env.TEST_CORE == 'true' }} run: | pwd ls -lahR $CUDA_CORE_ARTIFACTS_DIR - name: Download cuda.core Cython tests - if: ${{ inputs.test-core && env.SKIP_CYTHON_TEST == '0' }} + if: ${{ env.TEST_CORE == 'true' && env.SKIP_CYTHON_TEST == '0' }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: ${{ env.CUDA_CORE_ARTIFACT_NAME }}-tests @@ -288,13 +285,13 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} - name: Display structure of downloaded cuda.core Cython tests - if: ${{ inputs.test-core && env.SKIP_CYTHON_TEST == '0' }} + if: ${{ env.TEST_CORE == 'true' && env.SKIP_CYTHON_TEST == '0' }} run: | pwd ls -lahR $CUDA_CORE_CYTHON_TESTS_DIR - name: Download cuda.core test binaries - if: ${{ inputs.test-core }} + if: ${{ env.TEST_CORE == 'true' }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: ${{ env.CUDA_CORE_ARTIFACT_NAME }}-test-binaries @@ -303,7 +300,7 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} - name: Display structure of downloaded cuda.core test binaries - if: ${{ inputs.test-core }} + if: ${{ env.TEST_CORE == 'true' }} run: | pwd ls -lahR $CUDA_CORE_TEST_BINARIES_DIR @@ -319,7 +316,7 @@ jobs: AGENT_TOOLSDIRECTORY: "/opt/hostedtoolcache" - name: Enable Scientific Python Nightly Wheels for Python 3.15 - if: ${{ (inputs.test-bindings || inputs.test-core || inputs.test-python) && + if: ${{ (env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' || env.TEST_PYTHON == 'true') && startsWith(matrix.PY_VER, '3.15') }} run: | echo "PIP_EXTRA_INDEX_URL=https://pypi.anaconda.org/scientific-python-nightly-wheels/simple" >> "$GITHUB_ENV" @@ -334,7 +331,7 @@ jobs: cuda-version: ${{ matrix.CUDA_VER }} - name: Set up latest cuda_sanitizer_api - if: ${{ (inputs.test-bindings || inputs.test-core) && env.SETUP_SANITIZER == '1' }} + if: ${{ (env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true') && env.SETUP_SANITIZER == '1' }} uses: ./.github/actions/fetch_ctk continue-on-error: false with: @@ -343,7 +340,7 @@ jobs: cuda-components: "cuda_sanitizer_api" - name: Set up compute-sanitizer - if: ${{ inputs.test-bindings || inputs.test-core }} + if: ${{ env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' }} run: setup-sanitizer - name: Set up test repetition on nightly runs @@ -351,7 +348,7 @@ jobs: # ── Standard test steps (skipped for nightly modes) ── - name: Run cuda.pathfinder tests with see_what_works - if: ${{ inputs.test-mode == 'standard' && inputs.test-pathfinder }} + if: ${{ inputs.test-mode == 'standard' && env.TEST_PATHFINDER == 'true' }} env: CUDA_PATHFINDER_TEST_LOAD_NVIDIA_DYNAMIC_LIB_STRICTNESS: see_what_works CUDA_PATHFINDER_TEST_FIND_NVIDIA_HEADERS_STRICTNESS: see_what_works @@ -359,14 +356,14 @@ jobs: run: run-tests pathfinder - name: Run cuda.bindings tests - if: ${{ inputs.test-mode == 'standard' && inputs.test-bindings && env.SKIP_CUDA_BINDINGS_TEST == '0' }} + if: ${{ inputs.test-mode == 'standard' && env.TEST_BINDINGS == 'true' && env.SKIP_CUDA_BINDINGS_TEST == '0' }} env: CUDA_VER: ${{ matrix.CUDA_VER }} LOCAL_CTK: ${{ matrix.LOCAL_CTK }} run: run-tests bindings - name: Run cuda.bindings benchmarks (smoke test) - if: ${{ inputs.test-mode == 'standard' && inputs.test-bindings && env.SKIP_CUDA_BINDINGS_TEST == '0' }} + if: ${{ inputs.test-mode == 'standard' && env.TEST_BINDINGS == 'true' && env.SKIP_CUDA_BINDINGS_TEST == '0' }} run: | pip install pyperf pushd benchmarks/cuda_bindings @@ -374,20 +371,20 @@ jobs: popd - name: Run cuda.core tests - if: ${{ inputs.test-mode == 'standard' && inputs.test-core }} + if: ${{ inputs.test-mode == 'standard' && env.TEST_CORE == 'true' }} env: CUDA_VER: ${{ matrix.CUDA_VER }} LOCAL_CTK: ${{ matrix.LOCAL_CTK }} run: run-tests core - name: Ensure cuda-python installable - if: ${{ inputs.test-mode == 'standard' && inputs.test-python && env.BINDINGS_SOURCE == 'main' }} + if: ${{ inputs.test-mode == 'standard' && env.TEST_PYTHON == 'true' && env.BINDINGS_SOURCE == 'main' }} run: | # Package suites install their own dependencies. A metapackage-only # run has no preceding suite, so install the exact local internal # wheels in one transaction while resolving released dependencies # such as cuda-core from the package index. - if ${{ inputs.test-bindings || inputs.test-core }}; then + if ${{ env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' }}; then dependency_args=(--no-deps) else dependency_args=( @@ -402,7 +399,7 @@ jobs: pip install --only-binary=:all: "${dependency_args[@]}" "${python_requirements[@]}" - name: Install cuda.pathfinder extra wheels for testing - if: ${{ inputs.test-mode == 'standard' && inputs.test-pathfinder }} + if: ${{ inputs.test-mode == 'standard' && env.TEST_PATHFINDER == 'true' }} run: | set -euo pipefail pushd cuda_pathfinder @@ -411,7 +408,7 @@ jobs: popd - name: Run cuda.pathfinder tests with all_must_work - if: ${{ inputs.test-mode == 'standard' && inputs.test-pathfinder }} + if: ${{ inputs.test-mode == 'standard' && env.TEST_PATHFINDER == 'true' }} env: CUDA_PATHFINDER_TEST_LOAD_NVIDIA_DYNAMIC_LIB_STRICTNESS: all_must_work CUDA_PATHFINDER_TEST_FIND_NVIDIA_HEADERS_STRICTNESS: all_must_work diff --git a/.github/workflows/test-wheel-windows.yml b/.github/workflows/test-wheel-windows.yml index 04b290b1cd0..eb9d7ce9764 100644 --- a/.github/workflows/test-wheel-windows.yml +++ b/.github/workflows/test-wheel-windows.yml @@ -22,18 +22,10 @@ on: nruns: type: number default: 1 - test-pathfinder: - type: boolean - default: true - test-bindings: - type: boolean - default: true - test-core: - type: boolean - default: true - test-python: - type: boolean - default: true + workplan: + description: JSON workplan. An empty value tests everything. + type: string + default: "" run-id: description: > Workflow run ID to download artifacts from. @@ -96,6 +88,11 @@ jobs: echo "MATRIX=${MATRIX}" | tee --append "${GITHUB_OUTPUT}" test: + env: + TEST_PATHFINDER: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.pathfinder.needs_test }} + TEST_BINDINGS: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.bindings.needs_test }} + TEST_CORE: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.core.needs_test }} + TEST_PYTHON: ${{ inputs.workplan == '' || fromJSON(inputs.workplan).modules.python.needs_test }} name: Python ${{ matrix.PY_VER }}, CUDA ${{ matrix.CUDA_VER }} (${{ (matrix.LOCAL_CTK == '1' && 'local') || 'wheels' }}), GPU ${{ matrix.GPU }}${{ matrix.GPU_COUNT != '1' && format(' (x{0})', matrix.GPU_COUNT) || '' }} (${{ matrix.DRIVER_MODE }})${{ matrix.ENV.TORCH_VER && format(', {0}+{1}', matrix.ENV.TORCH_VER, matrix.ENV.TORCH_CUDA) || '' }}${{ matrix.ENV.MODE == 'nightly-numba-cuda' && ', latest' || '' }} timeout-minutes: 60 # The build stage could fail but we want the CI to keep moving. @@ -151,7 +148,7 @@ jobs: LOCAL_CTK: ${{ matrix.LOCAL_CTK }} PY_VER: ${{ matrix.PY_VER }} SHA: ${{ inputs.sha || github.sha }} - SKIP_BINDINGS_TEST_OVERRIDE: ${{ !inputs.test-bindings && '1' || '0' }} + SKIP_BINDINGS_TEST_OVERRIDE: ${{ env.TEST_BINDINGS != 'true' && '1' || '0' }} shell: bash --noprofile --norc -xeuo pipefail {0} run: ./ci/tools/env-vars test @@ -163,7 +160,7 @@ jobs: run: echo "$MATRIX_ENV" | jq -r 'to_entries[] | "\(.key)=\(.value)"' >> "$GITHUB_ENV" - name: Download cuda-pathfinder build artifacts - if: ${{ inputs.test-pathfinder || inputs.test-bindings || inputs.test-core || inputs.test-python }} + if: ${{ env.TEST_PATHFINDER == 'true' || env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' || env.TEST_PYTHON == 'true' }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: cuda-pathfinder-wheel @@ -172,7 +169,7 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} - name: Download cuda-python build artifacts - if: ${{ inputs.test-python && env.BINDINGS_SOURCE == 'main' }} + if: ${{ env.TEST_PYTHON == 'true' && env.BINDINGS_SOURCE == 'main' }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: cuda-python-wheel @@ -181,7 +178,7 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} - name: Download cuda.bindings build artifacts - if: ${{ (inputs.test-bindings || inputs.test-core || inputs.test-python) && + if: ${{ (env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' || env.TEST_PYTHON == 'true') && env.BINDINGS_SOURCE == 'main' }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: @@ -191,7 +188,7 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} - name: Download cuda-python & cuda.bindings build artifacts from the prior branch - if: ${{ (inputs.test-bindings || inputs.test-core || inputs.test-python) && + if: ${{ (env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' || env.TEST_PYTHON == 'true') && env.BINDINGS_SOURCE == 'backport' }} env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -208,7 +205,7 @@ jobs: mv $OLD_BASENAME/*.whl "${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}"/ rmdir $OLD_BASENAME - if ${{ inputs.test-python }}; then + if ${{ env.TEST_PYTHON == 'true' }}; then gh run download $LATEST_PRIOR_RUN_ID -p cuda-python-wheel -R NVIDIA/cuda-python ls -al cuda-python-wheel mv cuda-python-wheel/*.whl . @@ -216,20 +213,20 @@ jobs: fi - name: Display structure of downloaded cuda-python artifacts - if: ${{ inputs.test-python && env.BINDINGS_SOURCE != 'published' }} + if: ${{ env.TEST_PYTHON == 'true' && env.BINDINGS_SOURCE != 'published' }} run: | Get-Location Get-ChildItem cuda_python*.whl | Select-Object Mode, LastWriteTime, Length, FullName - name: Display structure of downloaded cuda.bindings artifacts - if: ${{ (inputs.test-bindings || inputs.test-core || inputs.test-python) && + if: ${{ (env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' || env.TEST_PYTHON == 'true') && env.BINDINGS_SOURCE != 'published' }} run: | Get-Location Get-ChildItem -Recurse -Force $env:CUDA_BINDINGS_ARTIFACTS_DIR | Select-Object Mode, LastWriteTime, Length, FullName - name: Download cuda.bindings Cython tests - if: ${{ inputs.test-bindings && env.SKIP_CYTHON_TEST == '0' }} + if: ${{ env.TEST_BINDINGS == 'true' && env.SKIP_CYTHON_TEST == '0' }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: ${{ env.CUDA_BINDINGS_ARTIFACT_NAME }}-tests @@ -238,13 +235,13 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} - name: Display structure of downloaded cuda.bindings Cython tests - if: ${{ inputs.test-bindings && env.SKIP_CYTHON_TEST == '0' }} + if: ${{ env.TEST_BINDINGS == 'true' && env.SKIP_CYTHON_TEST == '0' }} run: | Get-Location Get-ChildItem -Recurse -Force $env:CUDA_BINDINGS_CYTHON_TESTS_DIR | Select-Object Mode, LastWriteTime, Length, FullName - name: Download cuda.core build artifacts - if: ${{ inputs.test-core }} + if: ${{ env.TEST_CORE == 'true' }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: ${{ env.CUDA_CORE_ARTIFACT_NAME }} @@ -253,13 +250,13 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} - name: Display structure of downloaded cuda.core build artifacts - if: ${{ inputs.test-core }} + if: ${{ env.TEST_CORE == 'true' }} run: | Get-Location Get-ChildItem -Recurse -Force $env:CUDA_CORE_ARTIFACTS_DIR | Select-Object Mode, LastWriteTime, Length, FullName - name: Download cuda.core Cython tests - if: ${{ inputs.test-core && env.SKIP_CYTHON_TEST == '0' }} + if: ${{ env.TEST_CORE == 'true' && env.SKIP_CYTHON_TEST == '0' }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: ${{ env.CUDA_CORE_ARTIFACT_NAME }}-tests @@ -268,13 +265,13 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} - name: Display structure of downloaded cuda.core Cython tests - if: ${{ inputs.test-core && env.SKIP_CYTHON_TEST == '0' }} + if: ${{ env.TEST_CORE == 'true' && env.SKIP_CYTHON_TEST == '0' }} run: | Get-Location Get-ChildItem -Recurse -Force $env:CUDA_CORE_CYTHON_TESTS_DIR | Select-Object Mode, LastWriteTime, Length, FullName - name: Download cuda.core test binaries - if: ${{ inputs.test-core }} + if: ${{ env.TEST_CORE == 'true' }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: ${{ env.CUDA_CORE_ARTIFACT_NAME }}-test-binaries @@ -283,7 +280,7 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} - name: Display structure of downloaded cuda.core test binaries - if: ${{ inputs.test-core }} + if: ${{ env.TEST_CORE == 'true' }} run: | Get-Location Get-ChildItem -Recurse -Force $env:CUDA_CORE_TEST_BINARIES_DIR | Select-Object Mode, LastWriteTime, Length, FullName @@ -296,7 +293,7 @@ jobs: allow-prereleases: ${{ startsWith(matrix.PY_VER, '3.15') }} - name: Enable Scientific Python Nightly Wheels for Python 3.15 - if: ${{ (inputs.test-bindings || inputs.test-core || inputs.test-python) && + if: ${{ (env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' || env.TEST_PYTHON == 'true') && startsWith(matrix.PY_VER, '3.15') }} shell: bash --noprofile --norc -xeuo pipefail {0} run: | @@ -326,7 +323,7 @@ jobs: # ── Standard test steps (skipped for nightly modes) ── - name: Run cuda.pathfinder tests with see_what_works - if: ${{ inputs.test-mode == 'standard' && inputs.test-pathfinder }} + if: ${{ inputs.test-mode == 'standard' && env.TEST_PATHFINDER == 'true' }} env: CUDA_PATHFINDER_TEST_LOAD_NVIDIA_DYNAMIC_LIB_STRICTNESS: see_what_works CUDA_PATHFINDER_TEST_FIND_NVIDIA_HEADERS_STRICTNESS: see_what_works @@ -335,7 +332,7 @@ jobs: run: run-tests pathfinder - name: Run cuda.bindings tests - if: ${{ inputs.test-mode == 'standard' && inputs.test-bindings && env.SKIP_CUDA_BINDINGS_TEST == '0' }} + if: ${{ inputs.test-mode == 'standard' && env.TEST_BINDINGS == 'true' && env.SKIP_CUDA_BINDINGS_TEST == '0' }} env: CUDA_VER: ${{ matrix.CUDA_VER }} LOCAL_CTK: ${{ matrix.LOCAL_CTK }} @@ -343,7 +340,7 @@ jobs: run: run-tests bindings - name: Run cuda.core tests - if: ${{ inputs.test-mode == 'standard' && inputs.test-core }} + if: ${{ inputs.test-mode == 'standard' && env.TEST_CORE == 'true' }} env: CUDA_VER: ${{ matrix.CUDA_VER }} LOCAL_CTK: ${{ matrix.LOCAL_CTK }} @@ -351,13 +348,13 @@ jobs: run: run-tests core - name: Ensure cuda-python installable - if: ${{ inputs.test-mode == 'standard' && inputs.test-python && env.BINDINGS_SOURCE == 'main' }} + if: ${{ inputs.test-mode == 'standard' && env.TEST_PYTHON == 'true' && env.BINDINGS_SOURCE == 'main' }} run: | # Package suites install their own dependencies. A metapackage-only # run has no preceding suite, so install the exact local internal # wheels in one transaction while resolving released dependencies # such as cuda-core from the package index. - if ('${{ inputs.test-bindings || inputs.test-core }}' -eq 'true') { + if ('${{ env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' }}' -eq 'true') { $dependencyArgs = @('--no-deps') } else { $dependencyArgs = @( @@ -372,7 +369,7 @@ jobs: pip install --only-binary=:all: @dependencyArgs @pythonRequirements - name: Install cuda.pathfinder extra wheels for testing - if: ${{ inputs.test-mode == 'standard' && inputs.test-pathfinder }} + if: ${{ inputs.test-mode == 'standard' && env.TEST_PATHFINDER == 'true' }} shell: bash --noprofile --norc -xeuo pipefail {0} run: | pushd cuda_pathfinder @@ -381,7 +378,7 @@ jobs: popd - name: Run cuda.pathfinder tests with all_must_work - if: ${{ inputs.test-mode == 'standard' && inputs.test-pathfinder }} + if: ${{ inputs.test-mode == 'standard' && env.TEST_PATHFINDER == 'true' }} env: CUDA_PATHFINDER_TEST_LOAD_NVIDIA_DYNAMIC_LIB_STRICTNESS: all_must_work CUDA_PATHFINDER_TEST_FIND_NVIDIA_HEADERS_STRICTNESS: all_must_work diff --git a/ci/tools/compute_ci_plan.py b/ci/tools/compute_ci_plan.py new file mode 100644 index 00000000000..07d57570cea --- /dev/null +++ b/ci/tools/compute_ci_plan.py @@ -0,0 +1,241 @@ +#!/usr/bin/env python3 + +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Compute the CI build and test workplan for a pull request.""" + +from __future__ import annotations + +import argparse +import json +import subprocess +from pathlib import PurePosixPath + +MODULES = ("pathfinder", "bindings", "core", "python") +PACKAGE_MODULES = { + "cuda_pathfinder": "pathfinder", + "cuda_bindings": "bindings", + "cuda_core": "core", + "cuda_python": "python", +} + +# Source changes have different build and test consumers. In particular, +# cuda-python source needs a same-version bindings wheel, while a core-only +# change can reuse the baseline cuda-python wheel. +SOURCE_IMPACT = { + "pathfinder": (set(MODULES), set(MODULES)), + "bindings": ({"bindings", "core", "python"}, {"bindings", "core", "python"}), + "core": ({"core"}, {"core", "python"}), + "python": ({"bindings", "python"}, {"python"}), +} + +IGNORED_BASENAMES = {"AGENTS.md", "CLAUDE.md"} +IGNORED_PATHS = { + ".coveragerc", + ".gitignore", + ".pre-commit-config.yaml", + ".spdx-ignore", + "CONTRIBUTING.md", + "LICENSE", + "SECURITY.md", + "context7.json", + "greptile.json", + "pixi.lock", + "pixi.toml", + "pytest.ini", + "ruff.toml", + "benchmarks/cuda_bindings/pixi.lock", + "benchmarks/cuda_bindings/pixi.toml", + "ci/.ci-pipeline-regen.md", + "ci/ci-pipeline.svg", + "ci/cleanup-pr-previews", + "ci/tools/check_mempool_hygiene.py", + "ci/tools/check_pixi_cuda_version.py", + "ci/tools/check_release_notes.py", + "ci/tools/download-wheels", + "ci/tools/run_pytest_with_stack.py", + "ci/tools/validate-release-wheels", + "cuda_bindings/pixi.lock", + "cuda_bindings/pixi.toml", + "cuda_core/pixi.lock", + "cuda_core/pixi.toml", + "cuda_pathfinder/pixi.lock", + "cuda_pathfinder/pixi.toml", +} +IGNORED_PREFIXES = ( + ".agents/", + "benchmarks/cuda_core/", + "ci/tools/tests/", + "cuda_python_test_helpers/", + "toolshed/", +) + +ALL_TEST_PATHS = { + ".github/workflows/test-wheel-linux.yml", + ".github/workflows/test-wheel-windows.yml", + "ci/test-matrix.yml", + "ci/tools/configure_driver_mode.ps1", + "ci/tools/guess_latest.sh", + "ci/tools/install_gpu_driver.ps1", + "ci/tools/install_gpu_driver.sh", + "ci/tools/run-tests", + "ci/tools/setup-sanitizer", +} + +INDEPENDENT_GITHUB_PATHS = { + ".github/PULL_REQUEST_TEMPLATE.md", + ".github/RELEASE-core.md", + ".github/actionlint.yaml", + ".github/copy-pr-bot.yaml", + ".github/dependabot.yml", + ".github/labeler.yml", +} +INDEPENDENT_WORKFLOWS = { + "backport.yml", + "bandit.yml", + "build-docs.yml", + "ci-nightly.yml", + "ci-pixi-source-test.yml", + "cleanup-pr-previews.yml", + "coverage.yml", + "pr-auto-label.yml", + "pr-metadata-check.yml", + "release-cuda-pathfinder.yml", + "release-upload.yml", + "release.yml", + "security-suite.yml", + "triagelabel.yml", +} +INDEPENDENT_ACTIONS = {"doc_preview", "get_pr_number"} + + +def _is_independent(path: str) -> bool: + if path.startswith(IGNORED_PREFIXES): + return True + + if path in INDEPENDENT_GITHUB_PATHS or path.startswith(".github/ISSUE_TEMPLATE/"): + return True + + parts = PurePosixPath(path).parts + if len(parts) >= 3 and parts[:2] == (".github", "workflows"): + return parts[2] in INDEPENDENT_WORKFLOWS + if len(parts) >= 3 and parts[:2] == (".github", "actions"): + return parts[2] in INDEPENDENT_ACTIONS + return False + + +def compute_workplan( + paths: list[str], + *, + merge_base: str, + baseline_run_id: str, + baseline_sha: str, +) -> dict[str, object]: + """Return the final CI decisions for the supplied changed paths.""" + source_changes: set[str] = set() + test_changes: set[str] = set() + all_tests = False + force_all = not merge_base or not baseline_run_id or not baseline_sha + + if not force_all: + for path in paths: + path_parts = PurePosixPath(path).parts + if not path_parts or path in IGNORED_PATHS or path_parts[-1] in IGNORED_BASENAMES: + continue + + if path == "README.md": + # cuda_python/README.md is a tracked symlink to this sdist input. + source_changes.add("python") + continue + + module = PACKAGE_MODULES.get(path_parts[0]) + if module is not None and len(path_parts) > 1: + relative = path_parts[1:] + if relative[0] == "docs": + continue + if relative[0] in {"tests", "examples"} or (module == "core" and relative == ("pytest.ini",)): + test_changes.add(module) + else: + source_changes.add(module) + continue + + if path.startswith("cuda_python_test_helpers/cuda_python_test_helpers/"): + test_changes.update(("bindings", "core")) + elif path.startswith("benchmarks/cuda_bindings/"): + test_changes.add("bindings") + elif path in ALL_TEST_PATHS: + all_tests = True + elif not _is_independent(path): + force_all = True + + if force_all: + builds = set(MODULES) + tests = set(MODULES) + else: + builds: set[str] = set() + tests = set(MODULES) if all_tests else set(test_changes) + for module in source_changes: + build_impact, test_impact = SOURCE_IMPACT[module] + builds.update(build_impact) + tests.update(test_impact) + + modules = { + module: { + "needs_build": module in builds, + "needs_test": module in tests, + } + for module in MODULES + } + return { + "modules": modules, + "jobs": { + "platform_builds": bool(builds or tests), + "sdist_tests": bool(builds), + "wheel_tests": bool(tests), + "core_api_checks": force_all or "core" in source_changes, + }, + "merge_base": merge_base, + "baseline": { + "run_id": baseline_run_id if not force_all else "", + "sha": baseline_sha if not force_all else "", + }, + } + + +def _changed_paths(merge_base: str, head: str) -> list[str]: + result = subprocess.run( # noqa: S603 - argv is passed directly to git without a shell. + ["git", "diff", "--no-renames", "--name-only", "-z", f"{merge_base}...{head}"], # noqa: S607 + check=True, + stdout=subprocess.PIPE, + ) + return [path.decode("utf-8", errors="surrogateescape") for path in result.stdout.split(b"\0") if path] + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--merge-base", default="") + parser.add_argument("--head", default="HEAD") + parser.add_argument("--baseline-run-id", default="") + parser.add_argument("--baseline-sha", default="") + args = parser.parse_args() + + if bool(args.baseline_run_id) != bool(args.baseline_sha): + parser.error("baseline run ID and SHA must be supplied together") + if args.baseline_sha and args.baseline_sha != args.merge_base: + parser.error("baseline SHA must match the merge base") + + reusable_baseline = bool(args.merge_base and args.baseline_run_id) + paths = _changed_paths(args.merge_base, args.head) if reusable_baseline else [] + plan = compute_workplan( + paths, + merge_base=args.merge_base, + baseline_run_id=args.baseline_run_id, + baseline_sha=args.baseline_sha, + ) + print(json.dumps(plan, separators=(",", ":"), sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/ci/tools/tests/test_compute_ci_plan.py b/ci/tools/tests/test_compute_ci_plan.py new file mode 100644 index 00000000000..9be450d98e6 --- /dev/null +++ b/ci/tools/tests/test_compute_ci_plan.py @@ -0,0 +1,100 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import unittest + +from ci.tools.compute_ci_plan import compute_workplan + +ALL_MODULES = {"pathfinder", "bindings", "core", "python"} + + +def plan_for(*paths: str, baseline: bool = True) -> dict[str, object]: + return compute_workplan( + list(paths), + merge_base="base", + baseline_run_id="123" if baseline else "", + baseline_sha="base" if baseline else "", + ) + + +def selected(plan: dict[str, object], key: str) -> set[str]: + modules = plan["modules"] + assert isinstance(modules, dict) + return {name for name, decision in modules.items() if decision[key]} + + +class ComputeWorkplanTest(unittest.TestCase): + def test_path_impacts(self) -> None: + cases = { + "cuda_pathfinder/cuda/pathfinder/_loader.py": (ALL_MODULES, ALL_MODULES, False), + "cuda_bindings/cuda/bindings/driver.pyx": ( + {"bindings", "core", "python"}, + {"bindings", "core", "python"}, + False, + ), + "cuda_core/cuda/core/_device.py": ({"core"}, {"core", "python"}, True), + "cuda_python/pyproject.toml": ({"bindings", "python"}, {"python"}, False), + "README.md": ({"bindings", "python"}, {"python"}, False), + "cuda_pathfinder/tests/test_loader.py": (set(), {"pathfinder"}, False), + "cuda_bindings/examples/0_Introduction/vectorAddDrv.py": (set(), {"bindings"}, False), + "cuda_core/pytest.ini": (set(), {"core"}, False), + "cuda_core/tests/fixtures/pixi.toml": (set(), {"core"}, False), + "cuda_python/tests/test_import.py": (set(), {"python"}, False), + "cuda_python/pixi.toml": ({"bindings", "python"}, {"python"}, False), + "cuda_python_test_helpers/cuda_python_test_helpers/cuda_utils.py": ( + set(), + {"bindings", "core"}, + False, + ), + "ci/tools/run-tests": (set(), ALL_MODULES, False), + "ci/versions.yml": (ALL_MODULES, ALL_MODULES, True), + } + + for path, (builds, tests, core_api) in cases.items(): + with self.subTest(path=path): + plan = plan_for(path) + assert selected(plan, "needs_build") == builds + assert selected(plan, "needs_test") == tests + assert plan["jobs"]["core_api_checks"] == core_api + + def test_ignored_paths_select_no_work(self) -> None: + for path in ( + "cuda_core/docs/index.rst", + "cuda_core/pixi.toml", + "benchmarks/cuda_bindings/pixi.toml", + "benchmarks/cuda_bindings/AGENTS.md", + ".github/workflows/ci-pixi-source-test.yml", + "benchmarks/cuda_core/benchmark.py", + ): + with self.subTest(path=path): + plan = plan_for(path) + assert not selected(plan, "needs_build") + assert not selected(plan, "needs_test") + + def test_unknown_path_and_missing_baseline_force_all(self) -> None: + for plan in ( + plan_for("new-top-level-file"), + plan_for("new-area/pixi.toml"), + plan_for(".github/workflows/new-main-ci-workflow.yml"), + plan_for("cuda_core/docs/index.rst", baseline=False), + compute_workplan([], merge_base="base", baseline_run_id="123", baseline_sha=""), + ): + assert selected(plan, "needs_build") == ALL_MODULES + assert selected(plan, "needs_test") == ALL_MODULES + assert plan["jobs"]["core_api_checks"] + assert plan["baseline"] == {"run_id": "", "sha": ""} + + def test_mixed_changes_are_combined(self) -> None: + plan = plan_for("cuda_core/tests/test_device.py", "cuda_python/pyproject.toml") + assert selected(plan, "needs_build") == {"bindings", "python"} + assert selected(plan, "needs_test") == {"core", "python"} + assert plan["jobs"]["platform_builds"] + assert plan["jobs"]["sdist_tests"] + assert plan["jobs"]["wheel_tests"] + + +if __name__ == "__main__": + unittest.main()