diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 43b5be5a5c..fdfeff5686 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -154,6 +154,7 @@ umbrella doesn't watch, or operate independently of the rest of CI: | `pr_title_check.yml` | Fires on `pull_request.types: [edited]` so it re-runs when a PR title is edited without a code push. | | `codeql.yml` | Security scanner; weekly schedule + on every push/PR. | | `miri.yml` | Nightly Miri safety checks. | +| `publish_snapshot.yml` | Nightly SNAPSHOT jars to repository.apache.org; skips when main has not changed. `dry_run` dispatch. | | `stale.yml` | Daily stale-PR closer. | | `take.yml` | Issue-comment trigger for `take` / `untake`. | | `label_new_issues.yml` | Issue trigger to apply `requires-triage`. | diff --git a/.github/workflows/publish_snapshot.yml b/.github/workflows/publish_snapshot.yml new file mode 100644 index 0000000000..4a1681855a --- /dev/null +++ b/.github/workflows/publish_snapshot.yml @@ -0,0 +1,296 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Publishes SNAPSHOT jars for the current development version to the ASF +# snapshot repository at https://repository.apache.org/content/repositories/snapshots/ +# +# The jars bundle native libraries for linux/amd64 and linux/aarch64, built +# the same way the release builder does (see dev/release/comet-rm/Dockerfile): +# inside an Ubuntu 20.04 container so the library links against glibc 2.31 and +# loads on the older distributions that Spark container images are based on, +# and with the same baseline CPU targets as the release (`make core-*-libs`). +# +# Credentials come from the NEXUS_USER / NEXUS_PW repository secrets that ASF +# Infra provisions for snapshot publishing. The parent pom (org.apache:apache) +# already maps SNAPSHOT deploys to the `apache.snapshots.https` server, so no +# pom changes are needed. +# +# A `dry_run` dispatch builds everything, verifies the jars and uploads them as +# workflow artifacts without touching Nexus. It also works on forks. + +name: Publish Snapshot + +concurrency: + # Never let two publishes of the same snapshot version race each other. + group: ${{ github.workflow }} + cancel-in-progress: false + +on: + schedule: + # 03:00 UTC daily, after the nightly Miri run and before most of the + # working day in the Americas and Europe. + - cron: '0 3 * * *' + workflow_dispatch: + inputs: + dry_run: + description: 'Build and verify the jars, upload them as workflow artifacts, and skip the Nexus deploy' + type: boolean + default: false + +permissions: + contents: read + +env: + PROTOC_VERSION: 30.2 + +jobs: + changes: + name: Check for new commits + # The scheduled run must never publish from a fork. A dry run is allowed + # anywhere so the workflow can be exercised before it lands. + if: github.repository == 'apache/datafusion-comet' || inputs.dry_run + runs-on: ubuntu-slim + outputs: + publish: ${{ steps.check.outputs.publish }} + steps: + - uses: actions/checkout@v7 + - name: Skip the scheduled run when main has not changed + id: check + run: | + # A manual dispatch always builds. The scheduled run skips when HEAD + # predates the previous run, which keeps Nexus from accumulating + # identical snapshots. The slack covers scheduling jitter. If a + # nightly fails for an infrastructure reason and nothing lands the + # next day, trigger it manually. + if [ "$GITHUB_EVENT_NAME" != "schedule" ]; then + echo "publish=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + head_ts=$(git log -1 --format=%ct) + age=$(( $(date +%s) - head_ts )) + if [ "$age" -gt $(( 24 * 3600 + 1800 )) ]; then + echo "HEAD is $(( age / 3600 ))h old; nothing new to publish" + echo "publish=false" >> "$GITHUB_OUTPUT" + else + echo "publish=true" >> "$GITHUB_OUTPUT" + fi + + native: + name: Build native library (linux/${{ matrix.arch }}) + needs: changes + if: needs.changes.outputs.publish == 'true' + strategy: + fail-fast: true + matrix: + include: + - arch: amd64 + runner: ubuntu-24.04 + make_target: core-amd64-libs + protoc_arch: x86_64 + - arch: aarch64 + runner: ubuntu-24.04-arm + make_target: core-arm64-libs + protoc_arch: aarch_64 + runs-on: ${{ matrix.runner }} + # Same base image as dev/release/comet-rm/Dockerfile. See the header comment. + container: ubuntu:20.04 + timeout-minutes: 90 + env: + CC: gcc-10 + CXX: g++-10 + # Inside the container HOME is /github/home while root's passwd entry + # says /root, and rustup refuses to guess between them. Pin both homes so + # rustup, cargo and the cache below agree on one location. + CARGO_HOME: /github/home/.cargo + RUSTUP_HOME: /github/home/.rustup + # The hdfs-sys crate's build script locates a JDK. + JAVA_HOME: /usr/lib/jvm/default-java + steps: + - name: Install build dependencies + run: | + export DEBIAN_FRONTEND=noninteractive + apt-get update + apt-get install -y --no-install-recommends \ + ca-certificates curl unzip git build-essential gcc-10 g++-10 \ + clang llvm cmake pkg-config libssl-dev default-jdk-headless + - uses: actions/checkout@v7 + - name: Install protoc + run: | + curl -sSfL --retry 3 -o protoc.zip \ + "https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-${{ matrix.protoc_arch }}.zip" + unzip -q protoc.zip -d /usr/local + protoc --version + - name: Install Rust + run: | + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal + echo "$CARGO_HOME/bin" >> "$GITHUB_PATH" + - name: Cache Cargo registry + uses: actions/cache@v6 + with: + path: | + ${{ env.CARGO_HOME }}/registry + ${{ env.CARGO_HOME }}/git + key: snapshot-cargo-${{ matrix.arch }}-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }} + restore-keys: | + snapshot-cargo-${{ matrix.arch }}- + - name: Build + run: make ${{ matrix.make_target }} + - name: Check the library links against the release glibc baseline + # The container's default shell is sh, not bash. + shell: bash + run: | + # objdump lists every GLIBC_x.y version node the library needs. The + # release builder is Ubuntu 20.04 (glibc 2.31); anything newer means + # this job drifted away from the release image. + max=$(objdump -T native/target/release/libcomet.so \ + | grep -o 'GLIBC_[0-9.]*' | sort -uV | tail -1) + echo "Highest glibc symbol version required: $max" + if [ "$(printf '%s\n' "$max" GLIBC_2.31 | sort -V | tail -1)" != "GLIBC_2.31" ]; then + echo "libcomet.so requires a glibc newer than the release baseline" + exit 1 + fi + - uses: ./.github/actions/upload-artifact-retry + with: + name: libcomet-linux-${{ matrix.arch }} + path: native/target/release/libcomet.so + if-no-files-found: error + retention-days: 1 + + deploy: + name: Build and deploy snapshot jars + needs: native + runs-on: ubuntu-24.04 + timeout-minutes: 90 + steps: + - uses: actions/checkout@v7 + - name: Install JDK + uses: actions/setup-java@v4 + with: + distribution: 'zulu' + # Every variant is built with JDK 17, matching pr_build_linux.yml. + java-version: 17 + - name: Cache Maven dependencies + uses: actions/cache@v6 + with: + path: ~/.m2/repository + key: snapshot-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + snapshot-maven- + - name: Download native libraries + uses: ./.github/actions/download-artifact-retry + with: + pattern: libcomet-linux-* + path: native-libs + - name: Bootstrap Maven + uses: ./.github/actions/maven-bootstrap + - name: Configure Maven server credentials + if: ${{ !inputs.dry_run }} + run: | + # Credentials are read from the environment at deploy time rather + # than written into the file. + mkdir -p ~/.m2 + cat > ~/.m2/settings.xml <<'EOF' + + + + apache.snapshots.https + ${env.NEXUS_USER} + ${env.NEXUS_PW} + + + + EOF + - name: Build and ${{ inputs.dry_run && 'install' || 'deploy' }} + env: + NEXUS_USER: ${{ secrets.NEXUS_USER }} + NEXUS_PW: ${{ secrets.NEXUS_PW }} + GOAL: ${{ inputs.dry_run && 'install' || 'deploy' }} + run: | + set -euo pipefail + version=$(./mvnw -B -q help:evaluate -Dexpression=project.version -DforceStdout) + case "$version" in + *-SNAPSHOT) ;; + *) echo "Refusing to publish non-SNAPSHOT version $version"; exit 1 ;; + esac + echo "Publishing $version from $(git rev-parse --short HEAD)" + + check_native_libs() { + local jar=$1 lib + for lib in linux/amd64 linux/aarch64; do + # grep has to read the whole listing. Stopping at the first match + # (grep -q) leaves unzip killed by SIGPIPE, which `pipefail` then + # reports as a missing library. + if ! unzip -l "$jar" | grep -F "org/apache/comet/$lib/libcomet.so" > /dev/null; then + echo "$jar is missing $lib/libcomet.so" + return 1 + fi + done + } + + # One build per published variant. Spark 3.4 and 3.5 are published for + # Scala 2.12, which downstream benchmarks still use; 4.0 and 4.1 + # default to Scala 2.13. + variants=( + "-Pspark-3.4 -Pscala-2.12" + "-Pspark-3.5 -Pscala-2.12" + "-Pspark-4.0" + "-Pspark-4.1" + ) + # The native libraries live in spark/target/classes, so re-copy them + # after each clean; the release script + # (dev/release/build-release-comet.sh) places them the same way. + mkdir -p jars + for profiles in "${variants[@]}"; do + echo "::group::Build with $profiles" + ./mvnw -B -q clean + lib_dir=spark/target/classes/org/apache/comet/linux + mkdir -p "$lib_dir/amd64" "$lib_dir/aarch64" + cp native-libs/libcomet-linux-amd64/libcomet.so "$lib_dir/amd64/" + cp native-libs/libcomet-linux-aarch64/libcomet.so "$lib_dir/aarch64/" + # Package first so the jar can be checked before it is published. + # $profiles is a space-separated list of -P flags, so it must + # word-split. + # shellcheck disable=SC2086 + ./mvnw -B package -DskipTests $profiles + # Only the shaded plugin jar, not the sources or unshaded jars. + built=(spark/target/comet-spark-spark*_*-"$version".jar) + if [ "${#built[@]}" -ne 1 ] || [ ! -f "${built[0]}" ]; then + echo "Expected one plugin jar for $profiles, found: ${built[*]}" + exit 1 + fi + check_native_libs "${built[0]}" + cp "${built[0]}" jars/ + # The jar passed the check, so publish it. This build reuses the + # target directory the check ran against, and the parent pom pins + # project.build.outputTimestamp, so it re-creates the same jar. + # + # The root pom skips deploying the parent pom, but consumers need it + # to resolve the child poms, and the release publishes it. The user + # property overrides every module's setting. + # shellcheck disable=SC2086 + ./mvnw -B "$GOAL" -DskipTests -Dmaven.deploy.skip=false $profiles + echo "::endgroup::" + done + ls -l jars + - name: Upload jars + if: ${{ inputs.dry_run }} + uses: ./.github/actions/upload-artifact-retry + with: + name: comet-snapshot-jars + path: jars/*.jar + if-no-files-found: error + retention-days: 7 diff --git a/docs/source/user-guide/latest/installation.md b/docs/source/user-guide/latest/installation.md index a8d05955f0..753f62daac 100644 --- a/docs/source/user-guide/latest/installation.md +++ b/docs/source/user-guide/latest/installation.md @@ -75,8 +75,45 @@ Cloud Service Providers. -This documentation is for the current development version of Comet. Published jar files are only available for released versions. -To use this version of Comet, see [Building from source](source.md). +This documentation is for the current development version of Comet, which has not been released. Nightly snapshot +jar files for this version are published to the +[ASF snapshot repository](https://repository.apache.org/content/repositories/snapshots/org/apache/datafusion/) for the +amd64 and arm64 architectures for Linux. For Apple macOS, it is currently necessary to +[build from source](source.md). + +Snapshots are unreleased development builds provided for testing and evaluation only. They are not Apache releases, +have not been voted on, and should not be used in production. Older snapshots are removed from the repository +periodically. + +A new snapshot is published each night that new commits land on the `main` branch. Every snapshot carries the same +version, `$COMET_VERSION`, so Maven-based tooling resolves the most recent one automatically. The +[Publish Snapshot](https://github.com/apache/datafusion-comet/actions/workflows/publish_snapshot.yml) workflow log +records the commit each snapshot was built from. + +The following artifacts are published: + +- `comet-spark-spark3.4_2.12` +- `comet-spark-spark3.5_2.12` +- `comet-spark-spark4.0_2.13` +- `comet-spark-spark4.1_2.13` + +To download a snapshot jar, browse to the artifact directory in the snapshot repository, for example +[comet-spark-spark4.1_2.13/$COMET_VERSION](https://repository.apache.org/content/repositories/snapshots/org/apache/datafusion/comet-spark-spark4.1_2.13/$COMET_VERSION/), +and pick the jar with the newest timestamp. Then use it as described in +[Run Spark Shell with Comet enabled](#run-spark-shell-with-comet-enabled). + +Alternatively, let Spark resolve the newest snapshot directly: + +```shell +$SPARK_HOME/bin/spark-shell \ + --repositories https://repository.apache.org/content/repositories/snapshots/ \ + --packages org.apache.datafusion:comet-spark-spark4.1_2.13:$COMET_VERSION \ + --conf spark.plugins=org.apache.spark.CometPlugin \ + --conf spark.shuffle.manager=org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager \ + --conf spark.comet.explain.fallback.enabled=true \ + --conf spark.memory.offHeap.enabled=true \ + --conf spark.memory.offHeap.size=4g +```