From 892f029f111a8f45dbb61dd64e3206f88d52997f Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sun, 13 Sep 2026 10:12:05 -0600 Subject: [PATCH 1/6] ci: publish nightly SNAPSHOT jars to repository.apache.org Add a scheduled workflow that builds the Comet native library for linux/amd64 and linux/aarch64 inside an Ubuntu 20.04 container, matching the glibc baseline of the release builder, and deploys SNAPSHOT jars for Spark 3.4, 3.5, 4.0 and 4.1 to the ASF snapshot repository. The run skips when main has not changed since the previous night, and a dry_run dispatch builds and verifies the jars without touching Nexus so the workflow can be exercised on a fork. Document how to consume the snapshots in the installation guide. --- .github/workflows/README.md | 1 + .github/workflows/publish_snapshot.yml | 271 ++++++++++++++++++ docs/source/user-guide/latest/installation.md | 41 ++- 3 files changed, 311 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/publish_snapshot.yml diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 43b5be5a5c..fdfeff5686 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -154,6 +154,7 @@ umbrella doesn't watch, or operate independently of the rest of CI: | `pr_title_check.yml` | Fires on `pull_request.types: [edited]` so it re-runs when a PR title is edited without a code push. | | `codeql.yml` | Security scanner; weekly schedule + on every push/PR. | | `miri.yml` | Nightly Miri safety checks. | +| `publish_snapshot.yml` | Nightly SNAPSHOT jars to repository.apache.org; skips when main has not changed. `dry_run` dispatch. | | `stale.yml` | Daily stale-PR closer. | | `take.yml` | Issue-comment trigger for `take` / `untake`. | | `label_new_issues.yml` | Issue trigger to apply `requires-triage`. | diff --git a/.github/workflows/publish_snapshot.yml b/.github/workflows/publish_snapshot.yml new file mode 100644 index 0000000000..dfcdc0a119 --- /dev/null +++ b/.github/workflows/publish_snapshot.yml @@ -0,0 +1,271 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Publishes SNAPSHOT jars for the current development version to the ASF +# snapshot repository at https://repository.apache.org/content/repositories/snapshots/ +# +# The jars bundle native libraries for linux/amd64 and linux/aarch64, built +# the same way the release builder does (see dev/release/comet-rm/Dockerfile): +# inside an Ubuntu 20.04 container so the library links against glibc 2.31 and +# loads on the older distributions that Spark container images are based on, +# and with the same baseline CPU targets as the release (`make core-*-libs`). +# +# Credentials come from the NEXUS_USER / NEXUS_PW repository secrets that ASF +# Infra provisions for snapshot publishing. The parent pom (org.apache:apache) +# already maps SNAPSHOT deploys to the `apache.snapshots.https` server, so no +# pom changes are needed. +# +# A `dry_run` dispatch builds everything, verifies the jars and uploads them as +# workflow artifacts without touching Nexus. It also works on forks. + +name: Publish Snapshot + +concurrency: + # Never let two publishes of the same snapshot version race each other. + group: ${{ github.workflow }} + cancel-in-progress: false + +on: + schedule: + # 03:00 UTC daily, after the nightly Miri run and before most of the + # working day in the Americas and Europe. + - cron: '0 3 * * *' + workflow_dispatch: + inputs: + dry_run: + description: 'Build and verify the jars, upload them as workflow artifacts, and skip the Nexus deploy' + type: boolean + default: false + +permissions: + contents: read + +env: + PROTOC_VERSION: 30.2 + +jobs: + changes: + name: Check for new commits + # The scheduled run must never publish from a fork. A dry run is allowed + # anywhere so the workflow can be exercised before it lands. + if: github.repository == 'apache/datafusion-comet' || inputs.dry_run + runs-on: ubuntu-slim + outputs: + publish: ${{ steps.check.outputs.publish }} + steps: + - uses: actions/checkout@v7 + - name: Skip the scheduled run when main has not changed + id: check + run: | + # A manual dispatch always builds. The scheduled run skips when HEAD + # predates the previous run, which keeps Nexus from accumulating + # identical snapshots. The slack covers scheduling jitter. If a + # nightly fails for an infrastructure reason and nothing lands the + # next day, trigger it manually. + if [ "$GITHUB_EVENT_NAME" != "schedule" ]; then + echo "publish=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + head_ts=$(git log -1 --format=%ct) + age=$(( $(date +%s) - head_ts )) + if [ "$age" -gt $(( 24 * 3600 + 1800 )) ]; then + echo "HEAD is $(( age / 3600 ))h old; nothing new to publish" + echo "publish=false" >> "$GITHUB_OUTPUT" + else + echo "publish=true" >> "$GITHUB_OUTPUT" + fi + + native: + name: Build native library (linux/${{ matrix.arch }}) + needs: changes + if: needs.changes.outputs.publish == 'true' + strategy: + fail-fast: true + matrix: + include: + - arch: amd64 + runner: ubuntu-24.04 + make_target: core-amd64-libs + protoc_arch: x86_64 + - arch: aarch64 + runner: ubuntu-24.04-arm + make_target: core-arm64-libs + protoc_arch: aarch_64 + runs-on: ${{ matrix.runner }} + # Same base image as dev/release/comet-rm/Dockerfile. See the header comment. + container: ubuntu:20.04 + timeout-minutes: 90 + env: + CC: gcc-10 + CXX: g++-10 + steps: + - name: Install build dependencies + run: | + export DEBIAN_FRONTEND=noninteractive + apt-get update + apt-get install -y --no-install-recommends \ + ca-certificates curl unzip git build-essential gcc-10 g++-10 \ + clang llvm cmake pkg-config libssl-dev + - uses: actions/checkout@v7 + - name: Install protoc + run: | + curl -sSfL --retry 3 -o protoc.zip \ + "https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-${{ matrix.protoc_arch }}.zip" + unzip -q protoc.zip -d /usr/local + protoc --version + - name: Install Rust + run: | + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal + echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" + - name: Cache Cargo registry + uses: actions/cache@v6 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + key: snapshot-cargo-${{ matrix.arch }}-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }} + restore-keys: | + snapshot-cargo-${{ matrix.arch }}- + - name: Build + run: make ${{ matrix.make_target }} + - name: Check the library links against the release glibc baseline + run: | + # objdump lists every GLIBC_x.y version node the library needs. The + # release builder is Ubuntu 20.04 (glibc 2.31); anything newer means + # this job drifted away from the release image. + objdump -T native/target/release/libcomet.so \ + | grep -o 'GLIBC_[0-9.]*' | sort -uV | tail -1 | tee glibc.txt + if [ "$(sort -V glibc.txt <(echo GLIBC_2.31) | tail -1)" != "GLIBC_2.31" ]; then + echo "libcomet.so requires a glibc newer than the release baseline" + exit 1 + fi + - uses: ./.github/actions/upload-artifact-retry + with: + name: libcomet-linux-${{ matrix.arch }} + path: native/target/release/libcomet.so + if-no-files-found: error + retention-days: 1 + + deploy: + name: ${{ inputs.dry_run && 'Build snapshot jars (dry run)' || 'Deploy snapshot jars' }} + needs: native + runs-on: ubuntu-24.04 + timeout-minutes: 90 + steps: + - uses: actions/checkout@v7 + - name: Install JDKs + uses: actions/setup-java@v4 + with: + distribution: 'zulu' + # Spark 3.4 is built with JDK 11 and everything else with JDK 17, + # matching pr_build_linux.yml. setup-java exports JAVA_HOME_11_X64 + # and JAVA_HOME_17_X64; the last entry is the default. + java-version: | + 11 + 17 + - name: Cache Maven dependencies + uses: actions/cache@v6 + with: + path: ~/.m2/repository + key: snapshot-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + snapshot-maven- + - name: Download native libraries + uses: ./.github/actions/download-artifact-retry + with: + pattern: libcomet-linux-* + path: native-libs + - name: Bootstrap Maven + uses: ./.github/actions/maven-bootstrap + - name: Configure Maven server credentials + if: ${{ !inputs.dry_run }} + run: | + # Credentials are read from the environment at deploy time rather + # than written into the file. + mkdir -p ~/.m2 + cat > ~/.m2/settings.xml <<'EOF' + + + + apache.snapshots.https + ${env.NEXUS_USER} + ${env.NEXUS_PW} + + + + EOF + - name: Build and ${{ inputs.dry_run && 'install' || 'deploy' }} + env: + NEXUS_USER: ${{ secrets.NEXUS_USER }} + NEXUS_PW: ${{ secrets.NEXUS_PW }} + GOAL: ${{ inputs.dry_run && 'install' || 'deploy' }} + run: | + set -euo pipefail + version=$(./mvnw -B -q help:evaluate -Dexpression=project.version -DforceStdout) + case "$version" in + *-SNAPSHOT) ;; + *) echo "Refusing to publish non-SNAPSHOT version $version"; exit 1 ;; + esac + echo "Publishing $version from $(git rev-parse --short HEAD)" + + # One build per published variant. The native libraries live in + # spark/target/classes, so re-copy them after each clean; the release + # script (dev/release/build-release-comet.sh) places them the same way. + mkdir -p jars + while read -r jdk profiles; do + echo "::group::Build with $profiles" + ./mvnw -B -q clean + lib_dir=spark/target/classes/org/apache/comet/linux + mkdir -p "$lib_dir/amd64" "$lib_dir/aarch64" + cp native-libs/libcomet-linux-amd64/libcomet.so "$lib_dir/amd64/" + cp native-libs/libcomet-linux-aarch64/libcomet.so "$lib_dir/aarch64/" + # The root pom skips deploying the parent pom, but consumers need it + # to resolve the child poms, and the release publishes it. The user + # property overrides every module's setting. $profiles is a + # space-separated list of -P flags, so it must word-split. + # shellcheck disable=SC2086 + JAVA_HOME="$jdk" ./mvnw -B "$GOAL" -DskipTests -Dmaven.deploy.skip=false $profiles + cp spark/target/comet-spark-spark*.jar jars/ + echo "::endgroup::" + done < -This documentation is for the current development version of Comet. Published jar files are only available for released versions. -To use this version of Comet, see [Building from source](source.md). +This documentation is for the current development version of Comet, which has not been released. Nightly snapshot +jar files for this version are published to the +[ASF snapshot repository](https://repository.apache.org/content/repositories/snapshots/org/apache/datafusion/) for the +amd64 and arm64 architectures for Linux. For Apple macOS, it is currently necessary to +[build from source](source.md). + +Snapshots are unreleased development builds provided for testing and evaluation only. They are not Apache releases, +have not been voted on, and should not be used in production. Older snapshots are removed from the repository +periodically. + +A new snapshot is published each night that new commits land on the `main` branch. Every snapshot carries the same +version, `$COMET_VERSION`, so Maven-based tooling resolves the most recent one automatically. The +[Publish Snapshot](https://github.com/apache/datafusion-comet/actions/workflows/publish_snapshot.yml) workflow log +records the commit each snapshot was built from. + +The following artifacts are published: + +- `comet-spark-spark3.4_2.12` +- `comet-spark-spark3.5_2.12` +- `comet-spark-spark4.0_2.13` +- `comet-spark-spark4.1_2.13` + +To download a snapshot jar, browse to the artifact directory in the snapshot repository, for example +[comet-spark-spark4.1_2.13/$COMET_VERSION](https://repository.apache.org/content/repositories/snapshots/org/apache/datafusion/comet-spark-spark4.1_2.13/$COMET_VERSION/), +and pick the jar with the newest timestamp. Then use it as described in +[Run Spark Shell with Comet enabled](#run-spark-shell-with-comet-enabled). + +Alternatively, let Spark resolve the newest snapshot directly: + +```shell +$SPARK_HOME/bin/spark-shell \ + --repositories https://repository.apache.org/content/repositories/snapshots/ \ + --packages org.apache.datafusion:comet-spark-spark4.1_2.13:$COMET_VERSION \ + --conf spark.plugins=org.apache.spark.CometPlugin \ + --conf spark.shuffle.manager=org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager \ + --conf spark.comet.explain.fallback.enabled=true \ + --conf spark.memory.offHeap.enabled=true \ + --conf spark.memory.offHeap.size=4g +``` From 05f04df96784fd5ece2f9962e15a39440a151cec Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sun, 13 Sep 2026 10:25:57 -0600 Subject: [PATCH 2/6] ci: install a JDK for hdfs-sys and pin Cargo homes in the snapshot container --- .github/workflows/publish_snapshot.yml | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/.github/workflows/publish_snapshot.yml b/.github/workflows/publish_snapshot.yml index dfcdc0a119..18a42d3c51 100644 --- a/.github/workflows/publish_snapshot.yml +++ b/.github/workflows/publish_snapshot.yml @@ -112,6 +112,13 @@ jobs: env: CC: gcc-10 CXX: g++-10 + # Inside the container HOME is /github/home while root's passwd entry + # says /root, and rustup refuses to guess between them. Pin both homes so + # rustup, cargo and the cache below agree on one location. + CARGO_HOME: /github/home/.cargo + RUSTUP_HOME: /github/home/.rustup + # The hdfs-sys crate's build script locates a JDK. + JAVA_HOME: /usr/lib/jvm/default-java steps: - name: Install build dependencies run: | @@ -119,7 +126,7 @@ jobs: apt-get update apt-get install -y --no-install-recommends \ ca-certificates curl unzip git build-essential gcc-10 g++-10 \ - clang llvm cmake pkg-config libssl-dev + clang llvm cmake pkg-config libssl-dev default-jdk-headless - uses: actions/checkout@v7 - name: Install protoc run: | @@ -130,13 +137,13 @@ jobs: - name: Install Rust run: | curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal - echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" + echo "$CARGO_HOME/bin" >> "$GITHUB_PATH" - name: Cache Cargo registry uses: actions/cache@v6 with: path: | - ~/.cargo/registry - ~/.cargo/git + ${{ env.CARGO_HOME }}/registry + ${{ env.CARGO_HOME }}/git key: snapshot-cargo-${{ matrix.arch }}-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }} restore-keys: | snapshot-cargo-${{ matrix.arch }}- @@ -161,7 +168,7 @@ jobs: retention-days: 1 deploy: - name: ${{ inputs.dry_run && 'Build snapshot jars (dry run)' || 'Deploy snapshot jars' }} + name: Build and deploy snapshot jars needs: native runs-on: ubuntu-24.04 timeout-minutes: 90 @@ -239,7 +246,8 @@ jobs: # space-separated list of -P flags, so it must word-split. # shellcheck disable=SC2086 JAVA_HOME="$jdk" ./mvnw -B "$GOAL" -DskipTests -Dmaven.deploy.skip=false $profiles - cp spark/target/comet-spark-spark*.jar jars/ + # Only the shaded plugin jar, not the sources or unshaded jars. + cp spark/target/comet-spark-spark*_*-"$version".jar jars/ echo "::endgroup::" done < Date: Sun, 13 Sep 2026 10:44:08 -0600 Subject: [PATCH 3/6] ci: run the snapshot glibc check under bash --- .github/workflows/publish_snapshot.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/publish_snapshot.yml b/.github/workflows/publish_snapshot.yml index 18a42d3c51..659271427b 100644 --- a/.github/workflows/publish_snapshot.yml +++ b/.github/workflows/publish_snapshot.yml @@ -150,13 +150,16 @@ jobs: - name: Build run: make ${{ matrix.make_target }} - name: Check the library links against the release glibc baseline + # The container's default shell is sh, not bash. + shell: bash run: | # objdump lists every GLIBC_x.y version node the library needs. The # release builder is Ubuntu 20.04 (glibc 2.31); anything newer means # this job drifted away from the release image. - objdump -T native/target/release/libcomet.so \ - | grep -o 'GLIBC_[0-9.]*' | sort -uV | tail -1 | tee glibc.txt - if [ "$(sort -V glibc.txt <(echo GLIBC_2.31) | tail -1)" != "GLIBC_2.31" ]; then + max=$(objdump -T native/target/release/libcomet.so \ + | grep -o 'GLIBC_[0-9.]*' | sort -uV | tail -1) + echo "Highest glibc symbol version required: $max" + if [ "$(printf '%s\n' "$max" GLIBC_2.31 | sort -V | tail -1)" != "GLIBC_2.31" ]; then echo "libcomet.so requires a glibc newer than the release baseline" exit 1 fi From 9cdbbe193465681cb40ee5d7b310a0c4a6ce2d93 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sun, 13 Sep 2026 11:14:03 -0600 Subject: [PATCH 4/6] ci: keep Maven from consuming the snapshot variant list on stdin --- .github/workflows/publish_snapshot.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish_snapshot.yml b/.github/workflows/publish_snapshot.yml index 659271427b..411e2d5b56 100644 --- a/.github/workflows/publish_snapshot.yml +++ b/.github/workflows/publish_snapshot.yml @@ -236,9 +236,11 @@ jobs: # spark/target/classes, so re-copy them after each clean; the release # script (dev/release/build-release-comet.sh) places them the same way. mkdir -p jars + # Maven reads stdin, so it must not see the heredoc that feeds the + # loop or it swallows the remaining variants. while read -r jdk profiles; do echo "::group::Build with $profiles" - ./mvnw -B -q clean + ./mvnw -B -q clean < /dev/null lib_dir=spark/target/classes/org/apache/comet/linux mkdir -p "$lib_dir/amd64" "$lib_dir/aarch64" cp native-libs/libcomet-linux-amd64/libcomet.so "$lib_dir/amd64/" @@ -248,7 +250,7 @@ jobs: # property overrides every module's setting. $profiles is a # space-separated list of -P flags, so it must word-split. # shellcheck disable=SC2086 - JAVA_HOME="$jdk" ./mvnw -B "$GOAL" -DskipTests -Dmaven.deploy.skip=false $profiles + JAVA_HOME="$jdk" ./mvnw -B "$GOAL" -DskipTests -Dmaven.deploy.skip=false $profiles < /dev/null # Only the shaded plugin jar, not the sources or unshaded jars. cp spark/target/comet-spark-spark*_*-"$version".jar jars/ echo "::endgroup::" @@ -262,6 +264,12 @@ jobs: run: | set -euo pipefail status=0 + expected=4 + actual=$(find jars -name '*.jar' | wc -l) + if [ "$actual" -ne "$expected" ]; then + echo "Expected $expected jars, found $actual" + status=1 + fi for jar in jars/*.jar; do for lib in linux/amd64 linux/aarch64; do if ! unzip -l "$jar" | grep -q "org/apache/comet/$lib/libcomet.so"; then From 4ce0bd133ea5077620854d30e78acf473ec1a906 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 14 Sep 2026 11:10:06 -0600 Subject: [PATCH 5/6] ci: build every snapshot variant with JDK 17 #5897 dropped JDK 11 support: the Spark 3.4 profile now targets Java 17 and a Maven enforcer rule rejects anything older, so the Spark 3.4 build in this workflow would fail on the JDK 11 it was pinned to. --- .github/workflows/publish_snapshot.yml | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/.github/workflows/publish_snapshot.yml b/.github/workflows/publish_snapshot.yml index 411e2d5b56..9863d1301b 100644 --- a/.github/workflows/publish_snapshot.yml +++ b/.github/workflows/publish_snapshot.yml @@ -177,16 +177,12 @@ jobs: timeout-minutes: 90 steps: - uses: actions/checkout@v7 - - name: Install JDKs + - name: Install JDK uses: actions/setup-java@v4 with: distribution: 'zulu' - # Spark 3.4 is built with JDK 11 and everything else with JDK 17, - # matching pr_build_linux.yml. setup-java exports JAVA_HOME_11_X64 - # and JAVA_HOME_17_X64; the last entry is the default. - java-version: | - 11 - 17 + # Every variant is built with JDK 17, matching pr_build_linux.yml. + java-version: 17 - name: Cache Maven dependencies uses: actions/cache@v6 with: @@ -238,7 +234,7 @@ jobs: mkdir -p jars # Maven reads stdin, so it must not see the heredoc that feeds the # loop or it swallows the remaining variants. - while read -r jdk profiles; do + while read -r profiles; do echo "::group::Build with $profiles" ./mvnw -B -q clean < /dev/null lib_dir=spark/target/classes/org/apache/comet/linux @@ -250,15 +246,15 @@ jobs: # property overrides every module's setting. $profiles is a # space-separated list of -P flags, so it must word-split. # shellcheck disable=SC2086 - JAVA_HOME="$jdk" ./mvnw -B "$GOAL" -DskipTests -Dmaven.deploy.skip=false $profiles < /dev/null + ./mvnw -B "$GOAL" -DskipTests -Dmaven.deploy.skip=false $profiles < /dev/null # Only the shaded plugin jar, not the sources or unshaded jars. cp spark/target/comet-spark-spark*_*-"$version".jar jars/ echo "::endgroup::" done < Date: Mon, 14 Sep 2026 11:10:24 -0600 Subject: [PATCH 6/6] ci: check snapshot jars before publishing them The native-library check ran after the whole build loop, so with GOAL=deploy every jar was already uploaded by the time it ran. Package each variant, check it, and only then run the publishing goal for that variant. The second Maven invocation reuses the same target directory, and the parent pom pins project.build.outputTimestamp, so it re-creates a byte-identical jar; locally it took 20 seconds against a 75-second packaging run. The check itself could also report a missing library for a complete jar: `unzip -l | grep -q` leaves unzip killed by SIGPIPE once grep exits on the match, and `pipefail` turns that into a failure. Letting grep read the whole listing fixes it. Against a real 13 MB jar the old form failed 100 times out of 100 locally with PIPESTATUS=141 0, the new form none, and the new form still reports a library that is genuinely absent. Also guard the jar glob so a build that produces no plugin jar, or more than one, fails with a clear message instead of publishing something unchecked. That subsumes the separate count assertion. --- .github/workflows/publish_snapshot.yml | 82 +++++++++++++++----------- 1 file changed, 46 insertions(+), 36 deletions(-) diff --git a/.github/workflows/publish_snapshot.yml b/.github/workflows/publish_snapshot.yml index 9863d1301b..4a1681855a 100644 --- a/.github/workflows/publish_snapshot.yml +++ b/.github/workflows/publish_snapshot.yml @@ -228,54 +228,64 @@ jobs: esac echo "Publishing $version from $(git rev-parse --short HEAD)" - # One build per published variant. The native libraries live in - # spark/target/classes, so re-copy them after each clean; the release - # script (dev/release/build-release-comet.sh) places them the same way. + check_native_libs() { + local jar=$1 lib + for lib in linux/amd64 linux/aarch64; do + # grep has to read the whole listing. Stopping at the first match + # (grep -q) leaves unzip killed by SIGPIPE, which `pipefail` then + # reports as a missing library. + if ! unzip -l "$jar" | grep -F "org/apache/comet/$lib/libcomet.so" > /dev/null; then + echo "$jar is missing $lib/libcomet.so" + return 1 + fi + done + } + + # One build per published variant. Spark 3.4 and 3.5 are published for + # Scala 2.12, which downstream benchmarks still use; 4.0 and 4.1 + # default to Scala 2.13. + variants=( + "-Pspark-3.4 -Pscala-2.12" + "-Pspark-3.5 -Pscala-2.12" + "-Pspark-4.0" + "-Pspark-4.1" + ) + # The native libraries live in spark/target/classes, so re-copy them + # after each clean; the release script + # (dev/release/build-release-comet.sh) places them the same way. mkdir -p jars - # Maven reads stdin, so it must not see the heredoc that feeds the - # loop or it swallows the remaining variants. - while read -r profiles; do + for profiles in "${variants[@]}"; do echo "::group::Build with $profiles" - ./mvnw -B -q clean < /dev/null + ./mvnw -B -q clean lib_dir=spark/target/classes/org/apache/comet/linux mkdir -p "$lib_dir/amd64" "$lib_dir/aarch64" cp native-libs/libcomet-linux-amd64/libcomet.so "$lib_dir/amd64/" cp native-libs/libcomet-linux-aarch64/libcomet.so "$lib_dir/aarch64/" + # Package first so the jar can be checked before it is published. + # $profiles is a space-separated list of -P flags, so it must + # word-split. + # shellcheck disable=SC2086 + ./mvnw -B package -DskipTests $profiles + # Only the shaded plugin jar, not the sources or unshaded jars. + built=(spark/target/comet-spark-spark*_*-"$version".jar) + if [ "${#built[@]}" -ne 1 ] || [ ! -f "${built[0]}" ]; then + echo "Expected one plugin jar for $profiles, found: ${built[*]}" + exit 1 + fi + check_native_libs "${built[0]}" + cp "${built[0]}" jars/ + # The jar passed the check, so publish it. This build reuses the + # target directory the check ran against, and the parent pom pins + # project.build.outputTimestamp, so it re-creates the same jar. + # # The root pom skips deploying the parent pom, but consumers need it # to resolve the child poms, and the release publishes it. The user - # property overrides every module's setting. $profiles is a - # space-separated list of -P flags, so it must word-split. + # property overrides every module's setting. # shellcheck disable=SC2086 - ./mvnw -B "$GOAL" -DskipTests -Dmaven.deploy.skip=false $profiles < /dev/null - # Only the shaded plugin jar, not the sources or unshaded jars. - cp spark/target/comet-spark-spark*_*-"$version".jar jars/ + ./mvnw -B "$GOAL" -DskipTests -Dmaven.deploy.skip=false $profiles echo "::endgroup::" - done <