ci: retry the Maven distribution download in every job that runs mvnw

.github/actions/java-test/action.yaml

@@ -76,45 +76,9 @@ runs:
         restore-keys: |
           ${{ runner.os }}-java-maven-
 
-    # Maven itself is stored outside the dependency repository. Keep its cache
-    # independent of pom.xml changes and preserve the macOS cache workaround.
-    - name: Restore Maven distribution
-      id: maven-distribution
-      if: ${{ runner.os != 'macOS' }}
-      uses: actions/cache/restore@v5
-      with:
-        path: |
-          ~/.m2/wrapper/dists
-          /root/.m2/wrapper/dists
-        key: ${{ runner.os }}-${{ runner.arch }}-maven-wrapper-${{ hashFiles('.mvn/wrapper/maven-wrapper.properties') }}
-
-    # Retry only the wrapper download, never compilation or test execution.
-    # Delays use exponential backoff (10s, 20s, 40s) plus 0-4s of random jitter.
-    - name: Bootstrap Maven
-      shell: bash
-      run: |
-        for attempt in 1 2 3 4; do
-          if ./mvnw -B --version; then
-            break
-          fi
-          if [ "$attempt" -eq 4 ]; then
-            echo "::error::Maven bootstrap failed after $attempt attempts; tests were not started."
-            exit 1
-          fi
-          delay=$((10 * (1 << (attempt - 1)) + RANDOM % 5))
-          echo "::warning::Maven bootstrap attempt $attempt failed; retrying in ${delay}s."
-          sleep "$delay"
-        done
-
-    # Save a successful bootstrap even when the subsequent tests fail.
-    - name: Save Maven distribution
-      if: ${{ runner.os != 'macOS' && steps.maven-distribution.outputs.cache-hit != 'true' }}
-      uses: actions/cache/save@v5
-      with:
-        path: |
-          ~/.m2/wrapper/dists
-          /root/.m2/wrapper/dists
-        key: ${{ steps.maven-distribution.outputs.cache-primary-key }}
+    # Maven itself is already installed (with download retries and its own
+    # cache) by setup-builder / setup-macos-builder via maven-bootstrap, which
+    # every caller of this action runs first.
 
     - name: Run all tests
       shell: bash
@@ -137,8 +101,7 @@ runs:
     - name: Upload crash logs
       if: failure()
       # These three stay on the plain action rather than
-      # ../upload-artifact-retry: a local action calling another local action
-      # is untested in this repo. The two failure-only uploads run on jobs that
+      # ../upload-artifact-retry. The two failure-only uploads run on jobs that
       # are already red. The test-report upload also runs on green jobs, so it
       # is continue-on-error: a FinalizeArtifact 403 must not turn a passing test
       # run into a red check, and nothing downstream consumes the reports.

.github/actions/maven-bootstrap/action.yaml

@@ -15,27 +15,60 @@
 # specific language governing permissions and limitations
 # under the License.
 
-name: "Bootstrap Maven"
-description: >
-  Make ./mvnw usable, tolerating a flaky download of the Maven distribution
-  itself. Caches the distribution outside the dependency repository and retries
-  only the wrapper bootstrap, never compilation or test execution.
+name: Bootstrap Maven
+description: >-
+  Install the Maven distribution the wrapper points at, retrying the download
+  and caching the result, so a Maven Central hiccup does not fail a job before
+  it has built or tested anything.
+
+# The dependency caches cover .m2/repository, but the wrapper installs Maven
+# itself under .m2/wrapper/dists. On a fresh runner the first ./mvnw call
+# therefore still downloads apache-maven-*-bin.zip from Maven Central, and that
+# single request has failed with HTTP 403 and 429 often enough to matter.
+# Every job that runs ./mvnw (directly, via make, or via setup-spark-builder)
+# goes through setup-builder or setup-macos-builder, which end by calling this
+# action; ci.yml's preflight job calls it directly before the RAT check.
+#
+# Requires a checkout and a JDK on PATH. Runs from the repository root.
 
 runs:
   using: "composite"
   steps:
+    # The wrapper installs distributions under $MAVEN_USER_HOME, defaulting to
+    # the JVM's user.home, which comes from the passwd entry rather than $HOME.
+    # In the Linux job containers that is /root while $HOME is /github/home, so
+    # the cache action's `~` misses the installation; on a hosted runner it is
+    # /home/runner, and /root is unreadable. Listing both directories does not
+    # work either: actions/cache abandons the whole save when any listed path
+    # cannot be stat'ed, so hosted runners never saved anything. Ask the JVM
+    # for the one directory that applies and use it for both restore and save.
+    - name: Locate Maven distribution directory
+      id: maven-dists
+      shell: bash
+      run: |
+        java_cmd=java
+        if [ -n "${JAVA_HOME:-}" ]; then
+          java_cmd="$JAVA_HOME/bin/java"
+        fi
+        user_home=$("$java_cmd" -XshowSettings:properties -version 2>&1 | sed -n 's/^ *user\.home = //p')
+        dists="${MAVEN_USER_HOME:-${user_home:-$HOME}/.m2}/wrapper/dists"
+        echo "Maven distributions live under $dists"
+        echo "path=$dists" >> "$GITHUB_OUTPUT"
+
     # Maven itself is stored outside the dependency repository. Keep its cache
-    # independent of pom.xml changes and preserve the macOS cache workaround.
+    # independent of pom.xml changes.
     - name: Restore Maven distribution
       id: maven-distribution
+      # Temporarily disabled on macOS to work around
+      # https://github.com/actions/runner-images/issues/13341; macOS still
+      # gets the retry below, just not the cache.
       if: ${{ runner.os != 'macOS' }}
       uses: actions/cache/restore@v5
       with:
-        path: |
-          ~/.m2/wrapper/dists
-          /root/.m2/wrapper/dists
+        path: ${{ steps.maven-dists.outputs.path }}
         key: ${{ runner.os }}-${{ runner.arch }}-maven-wrapper-${{ hashFiles('.mvn/wrapper/maven-wrapper.properties') }}
 
+    # Retry only the wrapper download, never compilation or test execution.
     # Delays use exponential backoff (10s, 20s, 40s) plus 0-4s of random jitter.
     - name: Bootstrap Maven
       shell: bash
@@ -45,20 +78,18 @@ runs:
             break
           fi
           if [ "$attempt" -eq 4 ]; then
-            echo "::error::Maven bootstrap failed after $attempt attempts; the job was not started."
+            echo "::error::Maven bootstrap failed after $attempt attempts; nothing was built."
             exit 1
           fi
           delay=$((10 * (1 << (attempt - 1)) + RANDOM % 5))
           echo "::warning::Maven bootstrap attempt $attempt failed; retrying in ${delay}s."
           sleep "$delay"
         done
 
-    # Save a successful bootstrap even when the work that follows fails.
+    # Save a successful bootstrap even when the rest of the job fails.
     - name: Save Maven distribution
       if: ${{ runner.os != 'macOS' && steps.maven-distribution.outputs.cache-hit != 'true' }}
       uses: actions/cache/save@v5
       with:
-        path: |
-          ~/.m2/wrapper/dists
-          /root/.m2/wrapper/dists
+        path: ${{ steps.maven-dists.outputs.path }}
         key: ${{ steps.maven-distribution.outputs.cache-primary-key }}

.github/actions/setup-builder/action.yaml

@@ -56,3 +56,8 @@ runs:
         rustup toolchain install ${{inputs.rust-version}}
         rustup default ${{inputs.rust-version}}
         rustup component add rustfmt clippy
+
+    # Runs last so the JDK is on PATH. Retries the Maven distribution download
+    # and caches it; see .github/actions/maven-bootstrap/action.yaml.
+    - name: Bootstrap Maven
+      uses: ./.github/actions/maven-bootstrap

.github/actions/setup-macos-builder/action.yaml

@@ -78,3 +78,8 @@ runs:
         rustup toolchain install ${{inputs.rust-version}}
         rustup default ${{inputs.rust-version}}
         rustup component add rustfmt clippy
+
+    # Runs last so the JDK is on PATH. Retries the Maven distribution download
+    # and caches it; see .github/actions/maven-bootstrap/action.yaml.
+    - name: Bootstrap Maven
+      uses: ./.github/actions/maven-bootstrap

.github/workflows/README.md

@@ -249,11 +249,11 @@ built-in step retry. Use `./.github/actions/upload-artifact-retry` instead for
 any artifact a later job consumes: same inputs and outputs, three attempts,
 15s then 45s backoff. Attempts 2 and 3 force `overwrite: true`, so the name
 must belong to exactly one producer in the run (see above). The uploads inside
-`./.github/actions/java-test` stay on the plain action, since a local action
-calling another local action is untested here. Its two failure-only uploads run
-on jobs that are already red. Its test-report upload also runs on green jobs
-and is `continue-on-error: true`: nothing downstream consumes the reports, and
-a `FinalizeArtifact` 403 must not turn a passing test run into a red check.
+`./.github/actions/java-test` stay on the plain action. Its two failure-only
+uploads run on jobs that are already red. Its test-report upload also runs on
+green jobs and is `continue-on-error: true`: nothing downstream consumes the
+reports, and a `FinalizeArtifact` 403 must not turn a passing test run into a
+red check.
 
 **Artifact download.** `actions/download-artifact` has the same narrow retry
 list, so a `ListArtifacts` answered `(403) Forbidden: Error from intermediary`
@@ -284,15 +284,18 @@ the whole pipeline by hand.
 **Maven wrapper bootstrap.** `./mvnw` downloads the Maven distribution itself on
 a cold runner, and a blip from `repo.maven.apache.org` fails the job before
 anything is compiled. `./.github/actions/maven-bootstrap` caches that
-distribution under `~/.m2/wrapper/dists` (keyed on
+distribution under the wrapper's `.m2/wrapper/dists` (keyed on
 `.mvn/wrapper/maven-wrapper.properties`, not `pom.xml`) and retries
 `./mvnw --version` four times with exponential backoff. It retries only the
 bootstrap, never compilation or test execution.
 
-Any job whose first Maven use is a bare `./mvnw` needs this step before it.
-`./.github/actions/java-test` carries its own inline copy rather than calling
-the composite, because a local action invoking another local action is
-deliberately avoided here (see the artifact-upload note above).
+`./.github/actions/setup-builder` and `./.github/actions/setup-macos-builder`
+run it as their last step, once the JDK is on PATH, so every job that goes
+through either of them is covered without a step of its own; that includes
+the `java-test`, `rust-test` and `setup-spark-builder` callers. `preflight` in
+`ci.yml` uses no setup action and calls it directly before the RAT check. A
+new job that runs `./mvnw` without going through a setup action needs the
+step before its first Maven use.
 
 ## Merge queue
 

.github/workflows/ci.yml

@@ -88,6 +88,11 @@ jobs:
           distribution: temurin
           java-version: 17
 
+      # Preflight gates every other job, so a single failed download of the
+      # Maven distribution here would fail the whole run.
+      - name: Bootstrap Maven
+        uses: ./.github/actions/maven-bootstrap
+
       - name: Apache RAT license check
         run: ./mvnw -B -N apache-rat:check
 

.github/workflows/pr_build_linux.yml

@@ -138,9 +138,6 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-java-maven-
 
-      - name: Bootstrap Maven
-        uses: ./.github/actions/maven-bootstrap
-
       - name: Run scalafix check
         run: |
           ./mvnw -B package -DskipTests scalafix:scalafix -Dscalafix.mode=CHECK -Psemanticdb ${{ matrix.profile.maven_opts }}
@@ -195,9 +192,6 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-java-maven-
 
-      - name: Bootstrap Maven
-        uses: ./.github/actions/maven-bootstrap
-
       - name: Compile (skip tests)
         run: ./mvnw -B install -DskipTests -Dmaven.test.skip=true -Pspark-4.1
 
@@ -232,9 +226,6 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-java-maven-
 
-      - name: Bootstrap Maven
-        uses: ./.github/actions/maven-bootstrap
-
       - name: Verify reflected Celeborn internals
         env:
           SPARK_LOCAL_HOSTNAME: localhost
@@ -592,11 +583,6 @@ jobs:
           path: ./tpch
           key: tpch-${{ hashFiles('.github/workflows/pr_build_linux.yml') }}
 
-      # Without this, a transient repo.maven.apache.org blip fails the whole
-      # job before anything is compiled.
-      - name: Bootstrap Maven
-        uses: ./.github/actions/maven-bootstrap
-
       - name: Build project
         run: |
           ./mvnw -B -Prelease install -DskipTests
@@ -652,11 +638,6 @@ jobs:
           path: ./tpcds-sf-1
           key: tpcds-${{ hashFiles('.github/workflows/pr_build_linux.yml') }}
 
-      # Without this, a transient repo.maven.apache.org blip fails the whole
-      # job before anything is compiled.
-      - name: Bootstrap Maven
-        uses: ./.github/actions/maven-bootstrap
-
       - name: Build project
         run: |
           ./mvnw -B -Prelease install -DskipTests

dev/ci/check-ci-config.py

@@ -104,8 +104,9 @@
     # to nothing at all and merges having been exercised by no consumer.
     ([".github/actions/upload-artifact-retry/action.yaml"], BUILD_JOBS),
     ([".github/actions/download-artifact-retry/action.yaml"], BUILD_JOBS),
-    # The Maven bootstrap composite is called only from pr_build_linux.yml.
-    ([".github/actions/maven-bootstrap/action.yaml"], {"build_linux"}),
+    # Maven bootstrap runs inside setup-builder and setup-macos-builder, so it
+    # reaches every job that runs ./mvnw.
+    ([".github/actions/maven-bootstrap/action.yaml"], BUILD_JOBS),
     # Spot checks that the additions above did not widen unrelated routes.
     (["docs/source/user-guide/overview.md"], {"docs"}),
     (["native/core/benches/parquet_read.rs"], {"benchmark"}),

dev/ci/compute-changes.py

@@ -83,6 +83,7 @@
         ".github/actions/java-test/**",
         ".github/actions/upload-artifact-retry/**",
         ".github/actions/download-artifact-retry/**",
+        ".github/actions/maven-bootstrap/**",
         "!**.md",
         "!native/core/benches/**",
         "!native/spark-expr/benches/**",
@@ -131,6 +132,7 @@
         ".github/actions/setup-spark-builder/**",
         ".github/actions/upload-artifact-retry/**",
         ".github/actions/download-artifact-retry/**",
+        ".github/actions/maven-bootstrap/**",
         ".mvn/**",
         "mvnw",
     ],
@@ -158,6 +160,7 @@
         ".github/actions/setup-spark-builder/**",
         ".github/actions/upload-artifact-retry/**",
         ".github/actions/download-artifact-retry/**",
+        ".github/actions/maven-bootstrap/**",
         ".mvn/**",
         "mvnw",
     ],
@@ -185,6 +188,7 @@
         ".github/actions/setup-spark-builder/**",
         ".github/actions/upload-artifact-retry/**",
         ".github/actions/download-artifact-retry/**",
+        ".github/actions/maven-bootstrap/**",
         ".mvn/**",
         "mvnw",
     ],
@@ -212,6 +216,7 @@
         ".github/actions/setup-spark-builder/**",
         ".github/actions/upload-artifact-retry/**",
         ".github/actions/download-artifact-retry/**",
+        ".github/actions/maven-bootstrap/**",
         ".mvn/**",
         "mvnw",
     ],
@@ -241,6 +246,7 @@
         "dev/ci/test-iceberg-shards.py",
         ".github/actions/upload-artifact-retry/**",
         ".github/actions/download-artifact-retry/**",
+        ".github/actions/maven-bootstrap/**",
         ".mvn/**",
         "mvnw",
     ],
@@ -265,6 +271,7 @@
         "dev/ci/test-iceberg-shards.py",
         ".github/actions/upload-artifact-retry/**",
         ".github/actions/download-artifact-retry/**",
+        ".github/actions/maven-bootstrap/**",
         ".mvn/**",
         "mvnw",
     ],
@@ -289,6 +296,7 @@
         "dev/ci/test-iceberg-shards.py",
         ".github/actions/upload-artifact-retry/**",
         ".github/actions/download-artifact-retry/**",
+        ".github/actions/maven-bootstrap/**",
         ".mvn/**",
         "mvnw",
     ],
@@ -313,6 +321,7 @@
         "dev/ci/test-iceberg-shards.py",
         ".github/actions/upload-artifact-retry/**",
         ".github/actions/download-artifact-retry/**",
+        ".github/actions/maven-bootstrap/**",
         ".mvn/**",
         "mvnw",
     ],