From 7e84bac066846b0de0aa5564b4e0fc766747d009 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 12 Sep 2026 09:15:57 -0600 Subject: [PATCH 1/2] ci: retry the Maven distribution download in every job that runs mvnw The retry and cache added in #5422 only covered callers of the java-test action. Every other Maven caller (lint-java, build-spark-4-1, the benchmark verify jobs, setup-spark-builder, the Iceberg installs, preflight's RAT check) still downloads apache-maven-3.9.6-bin.zip on a cache miss with no retry, so one 403 from Maven Central fails the job before it builds anything. Move the restore/retry/save block into a bootstrap-maven composite action and call it at the end of setup-builder and setup-macos-builder, which every ./mvnw caller runs first, plus directly from preflight. Route edits to the new action to the same jobs as the setup actions. --- .github/actions/java-test/action.yaml | 45 ++----------------- .github/actions/maven-bootstrap/action.yaml | 30 +++++++++---- .github/actions/setup-builder/action.yaml | 5 +++ .../actions/setup-macos-builder/action.yaml | 5 +++ .github/workflows/README.md | 23 +++++----- .github/workflows/ci.yml | 5 +++ .github/workflows/pr_build_linux.yml | 19 -------- dev/ci/check-ci-config.py | 5 ++- dev/ci/compute-changes.py | 9 ++++ 9 files changed, 66 insertions(+), 80 deletions(-) diff --git a/.github/actions/java-test/action.yaml b/.github/actions/java-test/action.yaml index 42078b3c44d..2279a9d8210 100644 --- a/.github/actions/java-test/action.yaml +++ b/.github/actions/java-test/action.yaml @@ -76,45 +76,9 @@ runs: restore-keys: | ${{ runner.os }}-java-maven- - # Maven itself is stored outside the dependency repository. Keep its cache - # independent of pom.xml changes and preserve the macOS cache workaround. - - name: Restore Maven distribution - id: maven-distribution - if: ${{ runner.os != 'macOS' }} - uses: actions/cache/restore@v5 - with: - path: | - ~/.m2/wrapper/dists - /root/.m2/wrapper/dists - key: ${{ runner.os }}-${{ runner.arch }}-maven-wrapper-${{ hashFiles('.mvn/wrapper/maven-wrapper.properties') }} - - # Retry only the wrapper download, never compilation or test execution. - # Delays use exponential backoff (10s, 20s, 40s) plus 0-4s of random jitter. - - name: Bootstrap Maven - shell: bash - run: | - for attempt in 1 2 3 4; do - if ./mvnw -B --version; then - break - fi - if [ "$attempt" -eq 4 ]; then - echo "::error::Maven bootstrap failed after $attempt attempts; tests were not started." - exit 1 - fi - delay=$((10 * (1 << (attempt - 1)) + RANDOM % 5)) - echo "::warning::Maven bootstrap attempt $attempt failed; retrying in ${delay}s." - sleep "$delay" - done - - # Save a successful bootstrap even when the subsequent tests fail. - - name: Save Maven distribution - if: ${{ runner.os != 'macOS' && steps.maven-distribution.outputs.cache-hit != 'true' }} - uses: actions/cache/save@v5 - with: - path: | - ~/.m2/wrapper/dists - /root/.m2/wrapper/dists - key: ${{ steps.maven-distribution.outputs.cache-primary-key }} + # Maven itself is already installed (with download retries and its own + # cache) by setup-builder / setup-macos-builder via maven-bootstrap, which + # every caller of this action runs first. - name: Run all tests shell: bash @@ -137,8 +101,7 @@ runs: - name: Upload crash logs if: failure() # These three stay on the plain action rather than - # ../upload-artifact-retry: a local action calling another local action - # is untested in this repo. The two failure-only uploads run on jobs that + # ../upload-artifact-retry. The two failure-only uploads run on jobs that # are already red. The test-report upload also runs on green jobs, so it # is continue-on-error: a FinalizeArtifact 403 must not turn a passing test # run into a red check, and nothing downstream consumes the reports. diff --git a/.github/actions/maven-bootstrap/action.yaml b/.github/actions/maven-bootstrap/action.yaml index 7aab0b6030f..950de8a7d2c 100644 --- a/.github/actions/maven-bootstrap/action.yaml +++ b/.github/actions/maven-bootstrap/action.yaml @@ -15,19 +15,32 @@ # specific language governing permissions and limitations # under the License. -name: "Bootstrap Maven" -description: > - Make ./mvnw usable, tolerating a flaky download of the Maven distribution - itself. Caches the distribution outside the dependency repository and retries - only the wrapper bootstrap, never compilation or test execution. +name: Bootstrap Maven +description: >- + Install the Maven distribution the wrapper points at, retrying the download + and caching the result, so a Maven Central hiccup does not fail a job before + it has built or tested anything. + +# The dependency caches cover ~/.m2/repository, but the wrapper installs Maven +# itself under ~/.m2/wrapper/dists. On a fresh runner the first ./mvnw call +# therefore still downloads apache-maven-*-bin.zip from Maven Central, and that +# single request has failed with HTTP 403 and 429 often enough to matter. +# Every job that runs ./mvnw (directly, via make, or via setup-spark-builder) +# goes through setup-builder or setup-macos-builder, which end by calling this +# action; ci.yml's preflight job calls it directly before the RAT check. +# +# Requires a checkout and a JDK on PATH. Runs from the repository root. runs: using: "composite" steps: # Maven itself is stored outside the dependency repository. Keep its cache - # independent of pom.xml changes and preserve the macOS cache workaround. + # independent of pom.xml changes. - name: Restore Maven distribution id: maven-distribution + # Temporarily disabled on macOS to work around + # https://github.com/actions/runner-images/issues/13341; macOS still + # gets the retry below, just not the cache. if: ${{ runner.os != 'macOS' }} uses: actions/cache/restore@v5 with: @@ -36,6 +49,7 @@ runs: /root/.m2/wrapper/dists key: ${{ runner.os }}-${{ runner.arch }}-maven-wrapper-${{ hashFiles('.mvn/wrapper/maven-wrapper.properties') }} + # Retry only the wrapper download, never compilation or test execution. # Delays use exponential backoff (10s, 20s, 40s) plus 0-4s of random jitter. - name: Bootstrap Maven shell: bash @@ -45,7 +59,7 @@ runs: break fi if [ "$attempt" -eq 4 ]; then - echo "::error::Maven bootstrap failed after $attempt attempts; the job was not started." + echo "::error::Maven bootstrap failed after $attempt attempts; nothing was built." exit 1 fi delay=$((10 * (1 << (attempt - 1)) + RANDOM % 5)) @@ -53,7 +67,7 @@ runs: sleep "$delay" done - # Save a successful bootstrap even when the work that follows fails. + # Save a successful bootstrap even when the rest of the job fails. - name: Save Maven distribution if: ${{ runner.os != 'macOS' && steps.maven-distribution.outputs.cache-hit != 'true' }} uses: actions/cache/save@v5 diff --git a/.github/actions/setup-builder/action.yaml b/.github/actions/setup-builder/action.yaml index 0ccd01ad726..497c1d8b085 100644 --- a/.github/actions/setup-builder/action.yaml +++ b/.github/actions/setup-builder/action.yaml @@ -56,3 +56,8 @@ runs: rustup toolchain install ${{inputs.rust-version}} rustup default ${{inputs.rust-version}} rustup component add rustfmt clippy + + # Runs last so the JDK is on PATH. Retries the Maven distribution download + # and caches it; see .github/actions/maven-bootstrap/action.yaml. + - name: Bootstrap Maven + uses: ./.github/actions/maven-bootstrap diff --git a/.github/actions/setup-macos-builder/action.yaml b/.github/actions/setup-macos-builder/action.yaml index 7c1c8b522ed..155b849685c 100644 --- a/.github/actions/setup-macos-builder/action.yaml +++ b/.github/actions/setup-macos-builder/action.yaml @@ -78,3 +78,8 @@ runs: rustup toolchain install ${{inputs.rust-version}} rustup default ${{inputs.rust-version}} rustup component add rustfmt clippy + + # Runs last so the JDK is on PATH. Retries the Maven distribution download + # and caches it; see .github/actions/maven-bootstrap/action.yaml. + - name: Bootstrap Maven + uses: ./.github/actions/maven-bootstrap diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 43b5be5a5c0..7ccdd9513fc 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -249,11 +249,11 @@ built-in step retry. Use `./.github/actions/upload-artifact-retry` instead for any artifact a later job consumes: same inputs and outputs, three attempts, 15s then 45s backoff. Attempts 2 and 3 force `overwrite: true`, so the name must belong to exactly one producer in the run (see above). The uploads inside -`./.github/actions/java-test` stay on the plain action, since a local action -calling another local action is untested here. Its two failure-only uploads run -on jobs that are already red. Its test-report upload also runs on green jobs -and is `continue-on-error: true`: nothing downstream consumes the reports, and -a `FinalizeArtifact` 403 must not turn a passing test run into a red check. +`./.github/actions/java-test` stay on the plain action. Its two failure-only +uploads run on jobs that are already red. Its test-report upload also runs on +green jobs and is `continue-on-error: true`: nothing downstream consumes the +reports, and a `FinalizeArtifact` 403 must not turn a passing test run into a +red check. **Artifact download.** `actions/download-artifact` has the same narrow retry list, so a `ListArtifacts` answered `(403) Forbidden: Error from intermediary` @@ -284,15 +284,18 @@ the whole pipeline by hand. **Maven wrapper bootstrap.** `./mvnw` downloads the Maven distribution itself on a cold runner, and a blip from `repo.maven.apache.org` fails the job before anything is compiled. `./.github/actions/maven-bootstrap` caches that -distribution under `~/.m2/wrapper/dists` (keyed on +distribution under the wrapper's `.m2/wrapper/dists` (keyed on `.mvn/wrapper/maven-wrapper.properties`, not `pom.xml`) and retries `./mvnw --version` four times with exponential backoff. It retries only the bootstrap, never compilation or test execution. -Any job whose first Maven use is a bare `./mvnw` needs this step before it. -`./.github/actions/java-test` carries its own inline copy rather than calling -the composite, because a local action invoking another local action is -deliberately avoided here (see the artifact-upload note above). +`./.github/actions/setup-builder` and `./.github/actions/setup-macos-builder` +run it as their last step, once the JDK is on PATH, so every job that goes +through either of them is covered without a step of its own; that includes +the `java-test`, `rust-test` and `setup-spark-builder` callers. `preflight` in +`ci.yml` uses no setup action and calls it directly before the RAT check. A +new job that runs `./mvnw` without going through a setup action needs the +step before its first Maven use. ## Merge queue diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9904c2760b6..c2f06bb3ed4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -88,6 +88,11 @@ jobs: distribution: temurin java-version: 17 + # Preflight gates every other job, so a single failed download of the + # Maven distribution here would fail the whole run. + - name: Bootstrap Maven + uses: ./.github/actions/maven-bootstrap + - name: Apache RAT license check run: ./mvnw -B -N apache-rat:check diff --git a/.github/workflows/pr_build_linux.yml b/.github/workflows/pr_build_linux.yml index ff864ea7510..573adfa12f9 100644 --- a/.github/workflows/pr_build_linux.yml +++ b/.github/workflows/pr_build_linux.yml @@ -138,9 +138,6 @@ jobs: restore-keys: | ${{ runner.os }}-java-maven- - - name: Bootstrap Maven - uses: ./.github/actions/maven-bootstrap - - name: Run scalafix check run: | ./mvnw -B package -DskipTests scalafix:scalafix -Dscalafix.mode=CHECK -Psemanticdb ${{ matrix.profile.maven_opts }} @@ -195,9 +192,6 @@ jobs: restore-keys: | ${{ runner.os }}-java-maven- - - name: Bootstrap Maven - uses: ./.github/actions/maven-bootstrap - - name: Compile (skip tests) run: ./mvnw -B install -DskipTests -Dmaven.test.skip=true -Pspark-4.1 @@ -232,9 +226,6 @@ jobs: restore-keys: | ${{ runner.os }}-java-maven- - - name: Bootstrap Maven - uses: ./.github/actions/maven-bootstrap - - name: Verify reflected Celeborn internals env: SPARK_LOCAL_HOSTNAME: localhost @@ -592,11 +583,6 @@ jobs: path: ./tpch key: tpch-${{ hashFiles('.github/workflows/pr_build_linux.yml') }} - # Without this, a transient repo.maven.apache.org blip fails the whole - # job before anything is compiled. - - name: Bootstrap Maven - uses: ./.github/actions/maven-bootstrap - - name: Build project run: | ./mvnw -B -Prelease install -DskipTests @@ -652,11 +638,6 @@ jobs: path: ./tpcds-sf-1 key: tpcds-${{ hashFiles('.github/workflows/pr_build_linux.yml') }} - # Without this, a transient repo.maven.apache.org blip fails the whole - # job before anything is compiled. - - name: Bootstrap Maven - uses: ./.github/actions/maven-bootstrap - - name: Build project run: | ./mvnw -B -Prelease install -DskipTests diff --git a/dev/ci/check-ci-config.py b/dev/ci/check-ci-config.py index 6ccd22979ae..c3ba1f8b798 100644 --- a/dev/ci/check-ci-config.py +++ b/dev/ci/check-ci-config.py @@ -104,8 +104,9 @@ # to nothing at all and merges having been exercised by no consumer. ([".github/actions/upload-artifact-retry/action.yaml"], BUILD_JOBS), ([".github/actions/download-artifact-retry/action.yaml"], BUILD_JOBS), - # The Maven bootstrap composite is called only from pr_build_linux.yml. - ([".github/actions/maven-bootstrap/action.yaml"], {"build_linux"}), + # Maven bootstrap runs inside setup-builder and setup-macos-builder, so it + # reaches every job that runs ./mvnw. + ([".github/actions/maven-bootstrap/action.yaml"], BUILD_JOBS), # Spot checks that the additions above did not widen unrelated routes. (["docs/source/user-guide/overview.md"], {"docs"}), (["native/core/benches/parquet_read.rs"], {"benchmark"}), diff --git a/dev/ci/compute-changes.py b/dev/ci/compute-changes.py index 3380a1e4436..045655aaf96 100644 --- a/dev/ci/compute-changes.py +++ b/dev/ci/compute-changes.py @@ -83,6 +83,7 @@ ".github/actions/java-test/**", ".github/actions/upload-artifact-retry/**", ".github/actions/download-artifact-retry/**", + ".github/actions/maven-bootstrap/**", "!**.md", "!native/core/benches/**", "!native/spark-expr/benches/**", @@ -131,6 +132,7 @@ ".github/actions/setup-spark-builder/**", ".github/actions/upload-artifact-retry/**", ".github/actions/download-artifact-retry/**", + ".github/actions/maven-bootstrap/**", ".mvn/**", "mvnw", ], @@ -158,6 +160,7 @@ ".github/actions/setup-spark-builder/**", ".github/actions/upload-artifact-retry/**", ".github/actions/download-artifact-retry/**", + ".github/actions/maven-bootstrap/**", ".mvn/**", "mvnw", ], @@ -185,6 +188,7 @@ ".github/actions/setup-spark-builder/**", ".github/actions/upload-artifact-retry/**", ".github/actions/download-artifact-retry/**", + ".github/actions/maven-bootstrap/**", ".mvn/**", "mvnw", ], @@ -212,6 +216,7 @@ ".github/actions/setup-spark-builder/**", ".github/actions/upload-artifact-retry/**", ".github/actions/download-artifact-retry/**", + ".github/actions/maven-bootstrap/**", ".mvn/**", "mvnw", ], @@ -241,6 +246,7 @@ "dev/ci/test-iceberg-shards.py", ".github/actions/upload-artifact-retry/**", ".github/actions/download-artifact-retry/**", + ".github/actions/maven-bootstrap/**", ".mvn/**", "mvnw", ], @@ -265,6 +271,7 @@ "dev/ci/test-iceberg-shards.py", ".github/actions/upload-artifact-retry/**", ".github/actions/download-artifact-retry/**", + ".github/actions/maven-bootstrap/**", ".mvn/**", "mvnw", ], @@ -289,6 +296,7 @@ "dev/ci/test-iceberg-shards.py", ".github/actions/upload-artifact-retry/**", ".github/actions/download-artifact-retry/**", + ".github/actions/maven-bootstrap/**", ".mvn/**", "mvnw", ], @@ -313,6 +321,7 @@ "dev/ci/test-iceberg-shards.py", ".github/actions/upload-artifact-retry/**", ".github/actions/download-artifact-retry/**", + ".github/actions/maven-bootstrap/**", ".mvn/**", "mvnw", ], From d45ff1f4646e106665ed7ea69153fa1124497890 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 14 Sep 2026 09:01:22 -0600 Subject: [PATCH 2/2] ci: cache the Maven distribution under the directory the wrapper uses The cache steps listed both ~/.m2/wrapper/dists and /root/.m2/wrapper/dists so that one path list would cover the Linux job containers, where the wrapper installs under /root while $HOME is /github/home. On a hosted runner the wrapper installs under /home/runner and /root is unreadable, and actions/cache abandons the whole save as soon as one listed path fails to stat, so preflight downloaded Maven on every run and never saved it. Resolve the one directory that applies in a shell step, the same way the wrapper does (MAVEN_USER_HOME, else the JVM's user.home, which comes from the passwd entry rather than $HOME), and pass it to both restore and save. --- .github/actions/maven-bootstrap/action.yaml | 33 ++++++++++++++++----- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/.github/actions/maven-bootstrap/action.yaml b/.github/actions/maven-bootstrap/action.yaml index 950de8a7d2c..74dca64e612 100644 --- a/.github/actions/maven-bootstrap/action.yaml +++ b/.github/actions/maven-bootstrap/action.yaml @@ -21,8 +21,8 @@ description: >- and caching the result, so a Maven Central hiccup does not fail a job before it has built or tested anything. -# The dependency caches cover ~/.m2/repository, but the wrapper installs Maven -# itself under ~/.m2/wrapper/dists. On a fresh runner the first ./mvnw call +# The dependency caches cover .m2/repository, but the wrapper installs Maven +# itself under .m2/wrapper/dists. On a fresh runner the first ./mvnw call # therefore still downloads apache-maven-*-bin.zip from Maven Central, and that # single request has failed with HTTP 403 and 429 often enough to matter. # Every job that runs ./mvnw (directly, via make, or via setup-spark-builder) @@ -34,6 +34,27 @@ description: >- runs: using: "composite" steps: + # The wrapper installs distributions under $MAVEN_USER_HOME, defaulting to + # the JVM's user.home, which comes from the passwd entry rather than $HOME. + # In the Linux job containers that is /root while $HOME is /github/home, so + # the cache action's `~` misses the installation; on a hosted runner it is + # /home/runner, and /root is unreadable. Listing both directories does not + # work either: actions/cache abandons the whole save when any listed path + # cannot be stat'ed, so hosted runners never saved anything. Ask the JVM + # for the one directory that applies and use it for both restore and save. + - name: Locate Maven distribution directory + id: maven-dists + shell: bash + run: | + java_cmd=java + if [ -n "${JAVA_HOME:-}" ]; then + java_cmd="$JAVA_HOME/bin/java" + fi + user_home=$("$java_cmd" -XshowSettings:properties -version 2>&1 | sed -n 's/^ *user\.home = //p') + dists="${MAVEN_USER_HOME:-${user_home:-$HOME}/.m2}/wrapper/dists" + echo "Maven distributions live under $dists" + echo "path=$dists" >> "$GITHUB_OUTPUT" + # Maven itself is stored outside the dependency repository. Keep its cache # independent of pom.xml changes. - name: Restore Maven distribution @@ -44,9 +65,7 @@ runs: if: ${{ runner.os != 'macOS' }} uses: actions/cache/restore@v5 with: - path: | - ~/.m2/wrapper/dists - /root/.m2/wrapper/dists + path: ${{ steps.maven-dists.outputs.path }} key: ${{ runner.os }}-${{ runner.arch }}-maven-wrapper-${{ hashFiles('.mvn/wrapper/maven-wrapper.properties') }} # Retry only the wrapper download, never compilation or test execution. @@ -72,7 +91,5 @@ runs: if: ${{ runner.os != 'macOS' && steps.maven-distribution.outputs.cache-hit != 'true' }} uses: actions/cache/save@v5 with: - path: | - ~/.m2/wrapper/dists - /root/.m2/wrapper/dists + path: ${{ steps.maven-dists.outputs.path }} key: ${{ steps.maven-distribution.outputs.cache-primary-key }}