diff --git a/.github/actions/setup-deps/action.yml b/.github/actions/setup-deps/action.yml index ad966472a..649a51e4e 100644 --- a/.github/actions/setup-deps/action.yml +++ b/.github/actions/setup-deps/action.yml @@ -18,7 +18,7 @@ runs: # so we use ::group:: to make them visible. steps: - name: Setup spack - uses: spack/setup-spack@v2 + uses: spack/setup-spack@v3 with: ref: v1.1.0 color: true @@ -31,7 +31,37 @@ runs: spack compiler find echo "::endgroup::" + - name: Add FairMQ buildcache mirror + shell: spack-bash {0} + run: | + echo "::group::Add FairMQ buildcache mirror" + # Register the mirror globally (the env yamls only configure it inside the + # env, which is created *after* gcc is installed) so that the "Install GCC" + # step below can pull the compiler as a binary instead of building it from + # source (~58 min/job otherwise). + spack mirror add --unsigned --type binary \ + ghcr-buildcache oci://ghcr.io/fairrootgroup/fairmq-spack-buildcache + echo "::endgroup::" + + - name: Resolve lockfile + id: lock + shell: bash + run: | + # Prefer a committed, fully-concretized lockfile so the hashes match the + # binaries the buildcache pushed (guaranteed cache hits, no re-solve). + # Fresh runs (the buildcache itself) must re-concretize, so skip the lock. + lock="test/ci/locks/${{ inputs.env }}-gcc${{ inputs.gcc }}.lock" + if [ "${{ inputs.fresh }}" != "true" ] && [ -f "$lock" ]; then + echo "Using committed lockfile: $lock" + echo "use=true" >> "$GITHUB_OUTPUT" + echo "path=$lock" >> "$GITHUB_OUTPUT" + else + echo "No lockfile for ${{ inputs.env }}-gcc${{ inputs.gcc }}; concretizing from spec" + echo "use=false" >> "$GITHUB_OUTPUT" + fi + - name: Install GCC + if: steps.lock.outputs.use != 'true' shell: spack-bash {0} run: | echo "::group::Install GCC" @@ -41,6 +71,7 @@ runs: echo "::endgroup::" - name: Install dependencies + if: steps.lock.outputs.use != 'true' shell: spack-bash {0} run: | echo "::group::Install dependencies" @@ -48,5 +79,20 @@ runs: spack -e fairmq add gcc@${{ inputs.gcc }} spack -e fairmq config add "packages:all:require:'%gcc@${{ inputs.gcc }}'" spack -e fairmq install --fail-fast ${{ inputs.fresh == 'true' && '--fresh' || '' }} + echo "::endgroup::" + + - name: Install dependencies (from lockfile) + if: steps.lock.outputs.use == 'true' + shell: spack-bash {0} + run: | + echo "::group::Install dependencies (from lockfile)" + spack env create fairmq ${{ steps.lock.outputs.path }} + spack -e fairmq install --fail-fast + echo "::endgroup::" + + - name: Export environment + shell: spack-bash {0} + run: | + echo "::group::Export environment" spack env activate --sh fairmq | grep '^export ' | sed 's/^export //;s/;$//' >> $GITHUB_ENV echo "::endgroup::" diff --git a/.github/workflows/buildcache.yml b/.github/workflows/buildcache.yml index 19d251a03..26e03cbf4 100644 --- a/.github/workflows/buildcache.yml +++ b/.github/workflows/buildcache.yml @@ -5,6 +5,7 @@ on: schedule: - cron: '0 3 * * 0' # Weekly on Sunday at 3am UTC push: + branches: [dev, master] paths: - 'test/ci/spack-*.yaml' - '.github/workflows/buildcache.yml' @@ -18,7 +19,7 @@ jobs: build: if: github.repository == 'FairRootGroup/FairMQ' name: ${{ matrix.env }}-gcc-${{ matrix.gcc }} - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 permissions: packages: write strategy: @@ -49,10 +50,25 @@ jobs: spack -e fairmq mirror set --oci-username ${{ github.actor }} --oci-password-variable GITHUB_TOKEN ghcr-buildcache spack -e fairmq buildcache push --unsigned ghcr-buildcache + - name: Export lockfile + if: ${{ !cancelled() }} + shell: spack-bash {0} + run: | + mkdir -p test/ci/locks + cp "$(spack location -e fairmq)/spack.lock" \ + "test/ci/locks/${{ matrix.env }}-gcc${{ matrix.gcc }}.lock" + + - name: Upload lockfile + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v4 + with: + name: lock-${{ matrix.env }}-gcc${{ matrix.gcc }} + path: test/ci/locks/${{ matrix.env }}-gcc${{ matrix.gcc }}.lock + update-index: if: github.repository == 'FairRootGroup/FairMQ' && !cancelled() needs: build - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} permissions: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2a84d0b2a..bebcf6cd2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,7 +13,7 @@ jobs: build: if: github.repository == 'FairRootGroup/FairMQ' name: ${{ matrix.env }}-gcc-${{ matrix.gcc }} - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 strategy: fail-fast: false matrix: @@ -36,6 +36,12 @@ jobs: gcc: ${{ matrix.gcc }} env: ${{ matrix.env }} + - name: ccache + uses: hendrikmuhs/ccache-action@v1 + with: + key: ${{ github.job }}-${{ matrix.env }}-gcc${{ matrix.gcc }} + max-size: 500M + - name: Configure and Build uses: threeal/cmake-action@v2 with: @@ -44,6 +50,8 @@ jobs: CMAKE_BUILD_TYPE=RelWithDebInfo CMAKE_INSTALL_PREFIX=${{ github.workspace }}/install BUILD_TESTING=ON + CMAKE_C_COMPILER_LAUNCHER=ccache + CMAKE_CXX_COMPILER_LAUNCHER=ccache - name: Test uses: threeal/ctest-action@v1 @@ -56,7 +64,7 @@ jobs: sanitizers: if: github.repository == 'FairRootGroup/FairMQ' name: ${{ matrix.sanitizer.name }} - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 strategy: fail-fast: false matrix: @@ -83,6 +91,12 @@ jobs: with: gcc: '14' + - name: ccache + uses: hendrikmuhs/ccache-action@v1 + with: + key: ${{ github.job }}-${{ matrix.sanitizer.name }} + max-size: 500M + - name: Configure and Build uses: threeal/cmake-action@v2 with: @@ -92,6 +106,8 @@ jobs: options: | CMAKE_BUILD_TYPE=Debug BUILD_TESTING=ON + CMAKE_C_COMPILER_LAUNCHER=ccache + CMAKE_CXX_COMPILER_LAUNCHER=ccache ${{ matrix.sanitizer.options }} - name: Test @@ -102,7 +118,7 @@ jobs: static-analysis: if: github.repository == 'FairRootGroup/FairMQ' name: static-analysis - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v6 @@ -116,6 +132,12 @@ jobs: with: gcc: '14' + - name: ccache + uses: hendrikmuhs/ccache-action@v1 + with: + key: ${{ github.job }} + max-size: 500M + - name: Configure and Build uses: threeal/cmake-action@v2 with: @@ -124,6 +146,8 @@ jobs: CMAKE_BUILD_TYPE=Debug BUILD_TESTING=ON RUN_STATIC_ANALYSIS=ON + CMAKE_C_COMPILER_LAUNCHER=ccache + CMAKE_CXX_COMPILER_LAUNCHER=ccache - name: Check for warnings run: | diff --git a/test/ci/locks/README.md b/test/ci/locks/README.md new file mode 100644 index 000000000..1d81ebf1f --- /dev/null +++ b/test/ci/locks/README.md @@ -0,0 +1,21 @@ +# Committed spack lockfiles + +`setup-deps` installs from `-gcc.lock` here when a matching file exists, +skipping concretization so the resolved hashes match the binaries the buildcache +pushed (guaranteed cache hits). When no lockfile is present it falls back to +concretizing from `test/ci/spack-.yaml`. + +## Regenerating + +Lockfiles pin host externals (glibc, system gcc), so they must be concretized on +the same runner image CI uses (`ubuntu-24.04`) — do **not** generate them on a +local machine. + +1. Run the `Spack Buildcache` workflow (push to `dev`/`master` touching the spack + configs, or `workflow_dispatch`). It concretizes fresh (`fresh: 'true'`), + pushes binaries, and uploads each env's `spack.lock` as a `lock--gcc` + artifact. +2. Download the artifacts and commit them here as `-gcc.lock`. + +Regenerate whenever the specs in `test/ci/spack-*.yaml` change or the runner +image is bumped.