more efficient CI #1660
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| push: | |
| branches: | |
| - main | |
| pull_request: | |
| branches: | |
| - '**' | |
| env: | |
| GO_VERSION: 1.24.7 | |
| jobs: | |
| # Fast checks - no FFI needed, run immediately | |
| quick-checks: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Setup Go | |
| uses: actions/setup-go@v5 | |
| with: | |
| go-version: ${{ env.GO_VERSION }} | |
| cache: true | |
| cache-dependency-path: go.sum | |
| - name: Run checks in parallel | |
| run: | | |
| # gofmt check | |
| ( | |
| go fmt ./... | |
| git diff --quiet || { echo "gofmt needed"; exit 1; } | |
| ) & | |
| FMT_PID=$! | |
| # actionlint | |
| ( | |
| go install github.com/rhysd/actionlint/cmd/actionlint | |
| actionlint -shellcheck= -pyflakes= | |
| ) & | |
| LINT_PID=$! | |
| wait $FMT_PID || exit 1 | |
| wait $LINT_PID || exit 1 | |
| # Lint - runs immediately, no dependencies | |
| lint: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - name: Setup build environment | |
| uses: ./.github/actions/setup-build-env | |
| with: | |
| go-version: ${{ env.GO_VERSION }} | |
| - name: Install and run golangci-lint | |
| run: | | |
| curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v2.4.0 | |
| golangci-lint run -v --timeout 15m --concurrency 4 | |
| # Build variants - run in parallel, no dependencies | |
| build: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| variant: [debug, build, calibnet, 2k] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - name: Setup build environment | |
| uses: ./.github/actions/setup-build-env | |
| with: | |
| go-version: ${{ env.GO_VERSION }} | |
| - name: Build ${{ matrix.variant }} | |
| run: make ${{ matrix.variant }} | |
| # Unit tests - no database needed, runs immediately on self-hosted | |
| test-unit: | |
| runs-on: [self-hosted, docker] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - name: Setup build environment | |
| uses: ./.github/actions/setup-build-env | |
| with: | |
| go-version: ${{ env.GO_VERSION }} | |
| - name: Run unit tests | |
| run: go test -v --tags=debug -timeout 30m $(go list ./... | grep -v curio/itests) | |
| # Integration tests - parallel tests (use t.Parallel()) | |
| test-itest-parallel: | |
| runs-on: [self-hosted, docker] | |
| env: | |
| YB_CONTAINER: yugabyte-parallel-${{ github.run_id }} | |
| steps: | |
| - name: Check local proof parameters | |
| id: local-params | |
| run: | | |
| PARAMS_DIR="/var/tmp/filecoin-proof-parameters" | |
| if [ -d "$PARAMS_DIR" ] && [ "$(ls -A $PARAMS_DIR 2>/dev/null)" ]; then | |
| echo "exists=true" >> $GITHUB_OUTPUT | |
| else | |
| echo "exists=false" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Restore proof parameters from cache | |
| if: steps.local-params.outputs.exists != 'true' | |
| id: cache-params | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: /var/tmp/filecoin-proof-parameters | |
| key: proof-params-2k-v1 | |
| restore-keys: proof-params- | |
| - name: Fetch proof parameters | |
| if: steps.local-params.outputs.exists != 'true' && steps.cache-params.outputs.cache-hit != 'true' | |
| run: | | |
| echo "Fetching proof parameters (cache miss)..." | |
| lotus fetch-params 8388608 | |
| - name: Save proof parameters to cache | |
| if: steps.local-params.outputs.exists != 'true' && steps.cache-params.outputs.cache-hit != 'true' && github.ref == 'refs/heads/main' | |
| uses: actions/cache/save@v4 | |
| with: | |
| path: /var/tmp/filecoin-proof-parameters | |
| key: proof-params-2k-v1 | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - name: Start YugabyteDB | |
| id: start-yb | |
| run: | | |
| # Stop any existing container from previous runs | |
| docker stop $YB_CONTAINER 2>/dev/null || true | |
| docker rm $YB_CONTAINER 2>/dev/null || true | |
| # Start fresh container | |
| docker run --rm --name $YB_CONTAINER -d yugabytedb/yugabyte:2024.1.2.0-b77 bin/yugabyted start --daemon=false | |
| # Wait for it to be ready | |
| for i in {1..60}; do | |
| if docker exec $YB_CONTAINER bin/yugabyted status 2>/dev/null | grep -q Running; then | |
| echo "YugabyteDB is ready" | |
| break | |
| fi | |
| sleep 1 | |
| done | |
| YB_IP=$(docker inspect $YB_CONTAINER --format '{{ .NetworkSettings.Networks.bridge.IPAddress }}') | |
| echo "yb_ip=$YB_IP" >> $GITHUB_OUTPUT | |
| echo "YugabyteDB ready at $YB_IP" | |
| - name: Setup build environment | |
| uses: ./.github/actions/setup-build-env | |
| with: | |
| go-version: ${{ env.GO_VERSION }} | |
| - name: Run parallel integration tests | |
| env: | |
| CURIO_HARMONYDB_HOSTS: ${{ steps.start-yb.outputs.yb_ip }} | |
| LOTUS_HARMONYDB_HOSTS: ${{ steps.start-yb.outputs.yb_ip }} | |
| run: | | |
| echo "Using YugabyteDB at: $CURIO_HARMONYDB_HOSTS" | |
| # Serial tests require "serial" build tag, so they're excluded here | |
| go test -v --tags=debug -timeout 60m -parallel 4 ./itests/... | |
| - name: Stop YugabyteDB | |
| if: always() | |
| run: docker stop $YB_CONTAINER 2>/dev/null || true | |
| # Integration tests - serial tests (modify global state, cannot use t.Parallel()) | |
| test-itest-serial: | |
| runs-on: [self-hosted, docker] | |
| env: | |
| YB_CONTAINER: yugabyte-serial-${{ github.run_id }} | |
| steps: | |
| - name: Check local proof parameters | |
| id: local-params | |
| run: | | |
| PARAMS_DIR="/var/tmp/filecoin-proof-parameters" | |
| if [ -d "$PARAMS_DIR" ] && [ "$(ls -A $PARAMS_DIR 2>/dev/null)" ]; then | |
| echo "exists=true" >> $GITHUB_OUTPUT | |
| else | |
| echo "exists=false" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Restore proof parameters from cache | |
| if: steps.local-params.outputs.exists != 'true' | |
| id: cache-params | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: /var/tmp/filecoin-proof-parameters | |
| key: proof-params-2k-v1 | |
| restore-keys: proof-params- | |
| - name: Fetch proof parameters | |
| if: steps.local-params.outputs.exists != 'true' && steps.cache-params.outputs.cache-hit != 'true' | |
| run: | | |
| echo "Fetching proof parameters (cache miss)..." | |
| lotus fetch-params 8388608 | |
| - name: Save proof parameters to cache | |
| if: steps.local-params.outputs.exists != 'true' && steps.cache-params.outputs.cache-hit != 'true' && github.ref == 'refs/heads/main' | |
| uses: actions/cache/save@v4 | |
| with: | |
| path: /var/tmp/filecoin-proof-parameters | |
| key: proof-params-2k-v1 | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - name: Start YugabyteDB | |
| id: start-yb | |
| run: | | |
| # Stop any existing container from previous runs | |
| docker stop $YB_CONTAINER 2>/dev/null || true | |
| docker rm $YB_CONTAINER 2>/dev/null || true | |
| # Start fresh container | |
| docker run --rm --name $YB_CONTAINER -d yugabytedb/yugabyte:2024.1.2.0-b77 bin/yugabyted start --daemon=false | |
| # Wait for it to be ready | |
| for i in {1..60}; do | |
| if docker exec $YB_CONTAINER bin/yugabyted status 2>/dev/null | grep -q Running; then | |
| echo "YugabyteDB is ready" | |
| break | |
| fi | |
| sleep 1 | |
| done | |
| YB_IP=$(docker inspect $YB_CONTAINER --format '{{ .NetworkSettings.Networks.bridge.IPAddress }}') | |
| echo "yb_ip=$YB_IP" >> $GITHUB_OUTPUT | |
| echo "YugabyteDB ready at $YB_IP" | |
| - name: Setup build environment | |
| uses: ./.github/actions/setup-build-env | |
| with: | |
| go-version: ${{ env.GO_VERSION }} | |
| - name: Run serial integration tests | |
| env: | |
| CURIO_HARMONYDB_HOSTS: ${{ steps.start-yb.outputs.yb_ip }} | |
| LOTUS_HARMONYDB_HOSTS: ${{ steps.start-yb.outputs.yb_ip }} | |
| run: | | |
| echo "Using YugabyteDB at: $CURIO_HARMONYDB_HOSTS" | |
| # Tests in serial/ modify global state and cannot run in parallel | |
| go test -v --tags=debug,serial -timeout 30m ./itests/serial/... | |
| - name: Stop YugabyteDB | |
| if: always() | |
| run: docker stop $YB_CONTAINER 2>/dev/null || true | |
| # Gen check - verify generated code is up to date | |
| gen-check: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - name: Setup build environment | |
| uses: ./.github/actions/setup-build-env | |
| with: | |
| go-version: ${{ env.GO_VERSION }} | |
| - name: Install Go tools | |
| run: | | |
| go install golang.org/x/tools/cmd/goimports & | |
| go install github.com/hannahhoward/cbor-gen-for & | |
| go install github.com/swaggo/swag/cmd/swag & | |
| wait | |
| - name: api-gen | |
| run: make api-gen | |
| - name: go-generate | |
| run: make go-generate | |
| - name: cfgdoc-gen | |
| run: make cfgdoc-gen | |
| - name: docsgen (md + openrpc) | |
| run: make docsgen | |
| - name: marketgen | |
| run: make marketgen | |
| - name: docsgen-cli (builds curio + sptool) | |
| run: make docsgen-cli | |
| - name: fiximports + go mod tidy | |
| run: | | |
| go run ./scripts/fiximports | |
| go mod tidy | |
| - name: Check for changes | |
| run: | | |
| git diff --quiet || { git diff; exit 1; } | |
| # Supraseal build (kept separate - long running, different runner) | |
| build-supraseal-ubuntu24: | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - name: Free up disk space | |
| run: | | |
| sudo apt-get clean | |
| sudo rm -rf /usr/share/dotnet /opt/ghc "/usr/local/share/boost" "$AGENT_TOOLSDIRECTORY" | |
| - name: Install system dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y \ | |
| build-essential gcc-12 g++-12 nasm pkg-config \ | |
| autoconf automake libtool libssl-dev libnuma-dev \ | |
| uuid-dev libaio-dev libfuse3-dev libarchive-dev \ | |
| libkeyutils-dev libncurses-dev libgmp-dev libconfig++-dev \ | |
| python3 python3-pip python3-dev curl wget git xxd | |
| - name: Set up GCC 12 | |
| run: | | |
| sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 100 | |
| sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-12 100 | |
| - name: Cache CUDA installation | |
| id: cache-cuda | |
| uses: actions/cache@v4 | |
| with: | |
| path: /usr/local/cuda | |
| key: cuda-toolkit-ubuntu-24.04-v1 | |
| - name: Install CUDA Toolkit | |
| if: steps.cache-cuda.outputs.cache-hit != 'true' | |
| run: | | |
| wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64/cuda-keyring_1.1-1_all.deb | |
| sudo dpkg -i cuda-keyring_1.1-1_all.deb | |
| sudo apt-get update && sudo apt-get -y install cuda-toolkit | |
| - name: Set up CUDA environment | |
| run: | | |
| echo "/usr/local/cuda/bin" >> $GITHUB_PATH | |
| echo "CUDA_HOME=/usr/local/cuda" >> $GITHUB_ENV | |
| - name: Cache Python venv and SPDK | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| extern/supraseal/.venv | |
| extern/supraseal/deps/spdk-v24.05 | |
| key: supraseal-deps-ubuntu24-${{ hashFiles('extern/supraseal/build.sh') }} | |
| - name: Build Supraseal | |
| working-directory: extern/supraseal | |
| run: | | |
| export CC=gcc-12 CXX=g++-12 CUDA=/usr/local/cuda | |
| export PATH=/usr/local/cuda/bin:$PATH | |
| ./build.sh | |
| - name: Verify binaries | |
| run: | | |
| for bin in seal pc2 tree_r tree_r_cpu tree_d_cpu; do | |
| test -f extern/supraseal/bin/$bin || exit 1 | |
| done | |
| echo "✅ All Supraseal binaries built" |