From 045839efa9355c783dc63f0a0c2bbce316571eda Mon Sep 17 00:00:00 2001 From: Dennis Klein Date: Thu, 21 May 2026 15:50:06 +0200 Subject: [PATCH 1/4] ci: fetch gcc from buildcache mirror and pin runner image gcc was built from source (~58 min/job) because the FairMQ buildcache mirror is only configured inside the env yaml, while gcc is installed before the env is created. - register the mirror globally after spack setup so "Install GCC" pulls the compiler as a binary - pin runners to ubuntu-24.04 so the weekly buildcache and weekday CI share an image and concretize to matching hashes - bump setup-spack to v3 to match the update-index job --- .github/actions/setup-deps/action.yml | 14 +++++++++++++- .github/workflows/buildcache.yml | 4 ++-- .github/workflows/ci.yml | 6 +++--- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/.github/actions/setup-deps/action.yml b/.github/actions/setup-deps/action.yml index ad966472a..8075ac11c 100644 --- a/.github/actions/setup-deps/action.yml +++ b/.github/actions/setup-deps/action.yml @@ -18,7 +18,7 @@ runs: # so we use ::group:: to make them visible. steps: - name: Setup spack - uses: spack/setup-spack@v2 + uses: spack/setup-spack@v3 with: ref: v1.1.0 color: true @@ -31,6 +31,18 @@ runs: spack compiler find echo "::endgroup::" + - name: Add FairMQ buildcache mirror + shell: spack-bash {0} + run: | + echo "::group::Add FairMQ buildcache mirror" + # Register the mirror globally (the env yamls only configure it inside the + # env, which is created *after* gcc is installed) so that the "Install GCC" + # step below can pull the compiler as a binary instead of building it from + # source (~58 min/job otherwise). + spack mirror add --unsigned --type binary \ + ghcr-buildcache oci://ghcr.io/fairrootgroup/fairmq-spack-buildcache + echo "::endgroup::" + - name: Install GCC shell: spack-bash {0} run: | diff --git a/.github/workflows/buildcache.yml b/.github/workflows/buildcache.yml index 19d251a03..53be2482f 100644 --- a/.github/workflows/buildcache.yml +++ b/.github/workflows/buildcache.yml @@ -18,7 +18,7 @@ jobs: build: if: github.repository == 'FairRootGroup/FairMQ' name: ${{ matrix.env }}-gcc-${{ matrix.gcc }} - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 permissions: packages: write strategy: @@ -52,7 +52,7 @@ jobs: update-index: if: github.repository == 'FairRootGroup/FairMQ' && !cancelled() needs: build - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} permissions: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2a84d0b2a..c0878cba0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,7 +13,7 @@ jobs: build: if: github.repository == 'FairRootGroup/FairMQ' name: ${{ matrix.env }}-gcc-${{ matrix.gcc }} - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 strategy: fail-fast: false matrix: @@ -56,7 +56,7 @@ jobs: sanitizers: if: github.repository == 'FairRootGroup/FairMQ' name: ${{ matrix.sanitizer.name }} - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 strategy: fail-fast: false matrix: @@ -102,7 +102,7 @@ jobs: static-analysis: if: github.repository == 'FairRootGroup/FairMQ' name: static-analysis - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v6 From a25c9e61e7216e186e7751493cb4adcad634a616 Mon Sep 17 00:00:00 2001 From: Dennis Klein Date: Thu, 21 May 2026 15:51:18 +0200 Subject: [PATCH 2/4] ci: only run buildcache on dev/master pushes The push trigger had a path filter but no branch filter, so any PR-branch push touching those paths (e.g. a dependabot rebase pulling in setup-deps changes) launched the full fresh buildcache matrix concurrently with CI. - restrict the push trigger to branches [dev, master] - frees runners for CI; the cache still refreshes via cron and on dev/master --- .github/workflows/buildcache.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/buildcache.yml b/.github/workflows/buildcache.yml index 53be2482f..caea99595 100644 --- a/.github/workflows/buildcache.yml +++ b/.github/workflows/buildcache.yml @@ -5,6 +5,7 @@ on: schedule: - cron: '0 3 * * 0' # Weekly on Sunday at 3am UTC push: + branches: [dev, master] paths: - 'test/ci/spack-*.yaml' - '.github/workflows/buildcache.yml' From 58fcb82ac485cce16ac414f1288e9d2c80772e1c Mon Sep 17 00:00:00 2001 From: Dennis Klein Date: Sun, 31 May 2026 20:58:22 +0200 Subject: [PATCH 3/4] ci: cache fairmq compilation with ccache FairMQ's own sources (library, examples, tests) were recompiled from scratch in every matrix job on every push. - add hendrikmuhs/ccache-action to build, sanitizers and static-analysis jobs - set CMAKE_C/CXX_COMPILER_LAUNCHER=ccache so cmake routes through it - key the cache per (job, env, gcc) since ccache hashes the compiler --- .github/workflows/ci.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c0878cba0..bebcf6cd2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,6 +36,12 @@ jobs: gcc: ${{ matrix.gcc }} env: ${{ matrix.env }} + - name: ccache + uses: hendrikmuhs/ccache-action@v1 + with: + key: ${{ github.job }}-${{ matrix.env }}-gcc${{ matrix.gcc }} + max-size: 500M + - name: Configure and Build uses: threeal/cmake-action@v2 with: @@ -44,6 +50,8 @@ jobs: CMAKE_BUILD_TYPE=RelWithDebInfo CMAKE_INSTALL_PREFIX=${{ github.workspace }}/install BUILD_TESTING=ON + CMAKE_C_COMPILER_LAUNCHER=ccache + CMAKE_CXX_COMPILER_LAUNCHER=ccache - name: Test uses: threeal/ctest-action@v1 @@ -83,6 +91,12 @@ jobs: with: gcc: '14' + - name: ccache + uses: hendrikmuhs/ccache-action@v1 + with: + key: ${{ github.job }}-${{ matrix.sanitizer.name }} + max-size: 500M + - name: Configure and Build uses: threeal/cmake-action@v2 with: @@ -92,6 +106,8 @@ jobs: options: | CMAKE_BUILD_TYPE=Debug BUILD_TESTING=ON + CMAKE_C_COMPILER_LAUNCHER=ccache + CMAKE_CXX_COMPILER_LAUNCHER=ccache ${{ matrix.sanitizer.options }} - name: Test @@ -116,6 +132,12 @@ jobs: with: gcc: '14' + - name: ccache + uses: hendrikmuhs/ccache-action@v1 + with: + key: ${{ github.job }} + max-size: 500M + - name: Configure and Build uses: threeal/cmake-action@v2 with: @@ -124,6 +146,8 @@ jobs: CMAKE_BUILD_TYPE=Debug BUILD_TESTING=ON RUN_STATIC_ANALYSIS=ON + CMAKE_C_COMPILER_LAUNCHER=ccache + CMAKE_CXX_COMPILER_LAUNCHER=ccache - name: Check for warnings run: | From 1f561ae70ee35cd010d3d379d3a8f845773e5d95 Mon Sep 17 00:00:00 2001 From: Dennis Klein Date: Sun, 31 May 2026 21:08:42 +0200 Subject: [PATCH 4/4] ci: install deps from committed lockfiles when present Reusing concretization between the weekly buildcache (fresh) and weekday CI (reuse) can drift if runner externals change, causing avoidable cache misses. - setup-deps installs from test/ci/locks/-gcc.lock when it exists, skipping concretization for byte-identical hashes; falls back to the spec yaml otherwise - buildcache exports each env's spack.lock as a downloadable artifact so the lockfiles can be regenerated on the ubuntu-24.04 runner and committed - document the manual regeneration flow in test/ci/locks/README.md --- .github/actions/setup-deps/action.yml | 34 +++++++++++++++++++++++++++ .github/workflows/buildcache.yml | 15 ++++++++++++ test/ci/locks/README.md | 21 +++++++++++++++++ 3 files changed, 70 insertions(+) create mode 100644 test/ci/locks/README.md diff --git a/.github/actions/setup-deps/action.yml b/.github/actions/setup-deps/action.yml index 8075ac11c..649a51e4e 100644 --- a/.github/actions/setup-deps/action.yml +++ b/.github/actions/setup-deps/action.yml @@ -43,7 +43,25 @@ runs: ghcr-buildcache oci://ghcr.io/fairrootgroup/fairmq-spack-buildcache echo "::endgroup::" + - name: Resolve lockfile + id: lock + shell: bash + run: | + # Prefer a committed, fully-concretized lockfile so the hashes match the + # binaries the buildcache pushed (guaranteed cache hits, no re-solve). + # Fresh runs (the buildcache itself) must re-concretize, so skip the lock. + lock="test/ci/locks/${{ inputs.env }}-gcc${{ inputs.gcc }}.lock" + if [ "${{ inputs.fresh }}" != "true" ] && [ -f "$lock" ]; then + echo "Using committed lockfile: $lock" + echo "use=true" >> "$GITHUB_OUTPUT" + echo "path=$lock" >> "$GITHUB_OUTPUT" + else + echo "No lockfile for ${{ inputs.env }}-gcc${{ inputs.gcc }}; concretizing from spec" + echo "use=false" >> "$GITHUB_OUTPUT" + fi + - name: Install GCC + if: steps.lock.outputs.use != 'true' shell: spack-bash {0} run: | echo "::group::Install GCC" @@ -53,6 +71,7 @@ runs: echo "::endgroup::" - name: Install dependencies + if: steps.lock.outputs.use != 'true' shell: spack-bash {0} run: | echo "::group::Install dependencies" @@ -60,5 +79,20 @@ runs: spack -e fairmq add gcc@${{ inputs.gcc }} spack -e fairmq config add "packages:all:require:'%gcc@${{ inputs.gcc }}'" spack -e fairmq install --fail-fast ${{ inputs.fresh == 'true' && '--fresh' || '' }} + echo "::endgroup::" + + - name: Install dependencies (from lockfile) + if: steps.lock.outputs.use == 'true' + shell: spack-bash {0} + run: | + echo "::group::Install dependencies (from lockfile)" + spack env create fairmq ${{ steps.lock.outputs.path }} + spack -e fairmq install --fail-fast + echo "::endgroup::" + + - name: Export environment + shell: spack-bash {0} + run: | + echo "::group::Export environment" spack env activate --sh fairmq | grep '^export ' | sed 's/^export //;s/;$//' >> $GITHUB_ENV echo "::endgroup::" diff --git a/.github/workflows/buildcache.yml b/.github/workflows/buildcache.yml index caea99595..26e03cbf4 100644 --- a/.github/workflows/buildcache.yml +++ b/.github/workflows/buildcache.yml @@ -50,6 +50,21 @@ jobs: spack -e fairmq mirror set --oci-username ${{ github.actor }} --oci-password-variable GITHUB_TOKEN ghcr-buildcache spack -e fairmq buildcache push --unsigned ghcr-buildcache + - name: Export lockfile + if: ${{ !cancelled() }} + shell: spack-bash {0} + run: | + mkdir -p test/ci/locks + cp "$(spack location -e fairmq)/spack.lock" \ + "test/ci/locks/${{ matrix.env }}-gcc${{ matrix.gcc }}.lock" + + - name: Upload lockfile + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v4 + with: + name: lock-${{ matrix.env }}-gcc${{ matrix.gcc }} + path: test/ci/locks/${{ matrix.env }}-gcc${{ matrix.gcc }}.lock + update-index: if: github.repository == 'FairRootGroup/FairMQ' && !cancelled() needs: build diff --git a/test/ci/locks/README.md b/test/ci/locks/README.md new file mode 100644 index 000000000..1d81ebf1f --- /dev/null +++ b/test/ci/locks/README.md @@ -0,0 +1,21 @@ +# Committed spack lockfiles + +`setup-deps` installs from `-gcc.lock` here when a matching file exists, +skipping concretization so the resolved hashes match the binaries the buildcache +pushed (guaranteed cache hits). When no lockfile is present it falls back to +concretizing from `test/ci/spack-.yaml`. + +## Regenerating + +Lockfiles pin host externals (glibc, system gcc), so they must be concretized on +the same runner image CI uses (`ubuntu-24.04`) — do **not** generate them on a +local machine. + +1. Run the `Spack Buildcache` workflow (push to `dev`/`master` touching the spack + configs, or `workflow_dispatch`). It concretizes fresh (`fresh: 'true'`), + pushes binaries, and uploads each env's `spack.lock` as a `lock--gcc` + artifact. +2. Download the artifacts and commit them here as `-gcc.lock`. + +Regenerate whenever the specs in `test/ci/spack-*.yaml` change or the runner +image is bumped.