From 916eafea7c0e2fdb6fd9d332f352c36fb105cd27 Mon Sep 17 00:00:00 2001 From: MarcoGorelli <33491632+MarcoGorelli@users.noreply.github.com> Date: Thu, 5 Nov 2026 12:15:56 +0000 Subject: [PATCH 1/2] Add pandas asv example --- README.md | 2 + examples/pandas-asv/.gitignore | 4 + examples/pandas-asv/README.md | 118 +++++++++++++++++++++++++++ examples/pandas-asv/benchx.conf.json | 17 ++++ examples/pandas-asv/describe_env.py | 16 ++++ examples/pandas-asv/run.sh | 81 ++++++++++++++++++ 6 files changed, 238 insertions(+) create mode 100644 examples/pandas-asv/.gitignore create mode 100644 examples/pandas-asv/README.md create mode 100644 examples/pandas-asv/benchx.conf.json create mode 100644 examples/pandas-asv/describe_env.py create mode 100755 examples/pandas-asv/run.sh diff --git a/README.md b/README.md index 92377e3..de3f2c5 100644 --- a/README.md +++ b/README.md @@ -13,3 +13,5 @@ Currently, the following examples are provided: - `examples/matmul-params`: parametrized benchmarks to compare installed NumPy and PyTorch - `examples/numpy-venvs`: the same benchmarks under two NumPy versions, each in its own virtual environment, alternated round by round and compared. +- `examples/pandas-asv`: one benchmark from pandas' own asv suite, run unchanged with the asv + adapter under two pandas versions, each in its own virtual environment, and compared. diff --git a/examples/pandas-asv/.gitignore b/examples/pandas-asv/.gitignore new file mode 100644 index 0000000..603dcac --- /dev/null +++ b/examples/pandas-asv/.gitignore @@ -0,0 +1,4 @@ +.venv-*/ +out/ +pandas/ +__pycache__/ diff --git a/examples/pandas-asv/README.md b/examples/pandas-asv/README.md new file mode 100644 index 0000000..3b076d9 --- /dev/null +++ b/examples/pandas-asv/README.md @@ -0,0 +1,118 @@ +# pandas-asv: one pandas asv benchmark, two pandas versions + +One benchmark from pandas' own [asv](https://asv.readthedocs.io) suite, run unchanged by +`bx bench PATH --adapter asv` under two pandas versions, each in its own virtual environment, +alternated round by round, then compared. The asv adapter reads asv's benchmark format as a +language (discovery, `params`, `setup`, `setup_cache`, asv's benchmark version hash) and times it +with pyperf: neither asv nor asv_runner is installed. + +The suite is pandas' `asv_bench/` at one tag (`v3.1.1`), the same for both sides; only the +installed pandas differs. That is +[UC-04](https://github.com/benchx-dev/benchx/blob/main/docs/use-cases/UC-04-cross-env.md), +"fixed branch, comparing environments", driven the same way as `examples/numpy-venvs`. + +The case is `strings.Methods.time_lower('str')`: one parameter combination of one benchmark. +`dtype="str"` is object dtype in pandas 2.x and the pyarrow-backed string dtype from pandas 3.0, +so the two sides are expected to differ. + +## Run it + +You need [`uv`](https://docs.astral.sh/uv/), git, and network access the first time. You also +need benchx, installed in editable mode from a checkout, in the active environment. Then, from +this directory: + +```console +$ git clone https://github.com/benchx-dev/benchx.git +$ python3 -m venv .venv && . .venv/bin/activate # Python >= 3.11, or any environment you like +$ pip install -e benchx/prototype +$ ./run.sh +``` + +The script installs that same checkout in both venvs. It finds it from the benchx in the active +environment; `BENCHX_SRC=/prototype ./run.sh` names it instead. + +The tag, the two pandas versions, the case, the Python and the number of rounds are set at the +top of `run.sh`. Everything it creates is in `pandas/`, `.venv-a`, `.venv-b` and `out/`, all +git-ignored. A run takes under a minute after the venvs are cached. + +## What it does + +1. **Fetches the suite.** A shallow, sparse clone of pandas at `v3.1.1`, with only `asv_bench/` + checked out (a few MB). Nothing is built: the benchmarks import the pandas installed in each + venv, and each case runs from a scratch directory, so the unbuilt checkout cannot shadow it. +2. **Prepares two venvs**, `.venv-a` with pandas 2.3.3 and `.venv-b` with pandas 3.1.1, each + with pyarrow, pyperf and benchx. +3. **Runs the caller's loop.** For each of the 3 rounds, one call per side, in its venv: + + ```console + $ bx bench pandas/asv_bench --adapter asv "strings.Methods.time_lower('str')" \ + --run-key "$RUN_KEY" --label env=pandas-2.3.3 --round 0 --slot 0 --out "out/$RUN_KEY" + ``` + + `pandas/asv_bench` holds `asv.conf.json`, whose `benchmark_dir` is the suite; its checkout + is the target. The exact case name selects one parameter combination; `--filter REGEX` + (asv's `--bench`) would select by pattern instead. Modules the venv cannot import (no + matplotlib, scipy, ...) are listed on stderr and skipped. Each call prints a table: + + ``` + case median mean ± sd min .. max n status + ---------------------------------------- -------- ------------------- -------------------- -- ----------------------------------- + strings.Methods.time_lower [dtype='str'] 890.2 us 937.6 us ± 122.9 us 835.2 us .. 1.202 ms 10 success [setup-per-call-unbatched] + ``` + +4. **Compares.** `bx compare --run S --profile environments --label env --baseline pandas-2.3.3`: + + ``` + run pandas-1793878687 profile environments baseline pandas-2.3.3 contender pandas-3.1.1 + differs: coordinates.environment.system.deps.pandas, coordinates.environment.system.deps.sha256 + caveats: baseline: setup-per-call-unbatched, contender: setup-per-call-unbatched + + IMPROVED (1) + strings.Methods.time_lower wall-time effect -85.40% noise 1.76% rounds=3 + ``` + + (The numbers vary from run to run.) +5. **Lists the series.** One per environment, because the collected pandas version is part of + a series' identity. Every point carries the suite's revision (pandas `v3.1.1`), whichever + pandas it ran under: the target is the checkout the benchmarks come from, and what varies is + the environment, recorded by the collector. + +## How it is configured + +`benchx.conf.json` is found by walking up from the working directory, so `bx` reads it when +`run.sh` runs from here: + +- `adapter`: the harness (`asv`) and its precision, in pyperf's terms: 2 processes (asv's + rounds) × 5 values (asv's repeat), 1 warmup, loops calibrated. Flags such as `--processes`, + `--values`, `--min-time` and `--loops` override them per call. What neither gives comes from + each benchmark's own `repeat`, `number`, `sample_time` and `warmup_time`. +- `collectors.deps`: `describe_env.py`, a **context collector**. benchx runs it once per call, + in the environment the call runs in, and it prints the pandas, numpy and pyarrow it imports. + That lands in each result's `environment.system.deps`, which is how the `environments` + comparison knows what differs between the sides. +- `paths`: the store and the results go in `out/`. + +Each result also records the installed project's version (`provenance.info.asv.installed`) and +asv's hash of the benchmark's code (`protocol.asv.benchmark_version`), so editing the benchmark +starts a new series. + +## Caveats + +- **`setup-per-call-unbatched`.** `Methods` has a `setup`, so, as in asv_runner >= 0.3.0, every + timed call sees fresh state: one call per loop, setup re-run between calls outside the timer. + For a ~100 µs call that timing includes some timer overhead, and the result is flagged. A + benchmark whose calls leave the set-up state intact can declare `setup_scope = "sample"`. +- **Memory benchmarks** (`peakmem_*`, `mem_*`) and other non-`time_*` types are listed and + skipped. +- **Unstable boot time.** pyperf's `boot_time` metadata is part of `environment.system`, which + the `environments` profile uses to tell sides apart. On hosts where the kernel's boot time + drifts (seen on WSL2), calls with the same label can then become separate sides, and the + comparison fails the `two-sides` invariant. This is a benchx issue, not specific to asv. + +## What is in this directory + +| File | Role | +|---|---| +| `run.sh` | the driver: the suite, the venvs, the loop, the comparison | +| `benchx.conf.json` | the project file: the adapter and its precision, the `deps` collector, the paths | +| `describe_env.py` | the collector: prints the pandas, numpy and pyarrow this Python imports | diff --git a/examples/pandas-asv/benchx.conf.json b/examples/pandas-asv/benchx.conf.json new file mode 100644 index 0000000..ba18fdf --- /dev/null +++ b/examples/pandas-asv/benchx.conf.json @@ -0,0 +1,17 @@ +{ + "schema_version": "benchx/config/0.1.0", + "project": "pandas", + "adapter": { + "name": "asv", + "precision": { + "repetitions": { "mode": "fixed", "levels": [{ "unit": "process", "n": 2 }, { "unit": "value", "n": 5 }] }, + "warmup": { "mode": "count", "n_warmup": 1 } + } + }, + "collectors": { + "deps": { + "command": ["python", "{root}/describe_env.py"] + } + }, + "paths": { "store": "out/store.parquet", "results": "out" } +} diff --git a/examples/pandas-asv/describe_env.py b/examples/pandas-asv/describe_env.py new file mode 100644 index 0000000..2650951 --- /dev/null +++ b/examples/pandas-asv/describe_env.py @@ -0,0 +1,16 @@ +"""The collector: print the pandas this Python imports, and the libraries that decide how +fast it is. + +benchx.conf.json runs it once per call, in the environment the call runs in, so the two sides +can only differ here if their environments do. It prints the same text for the same +environment: no paths, no timestamps. +""" + +import json + +import numpy +import pandas +import pyarrow + +print(json.dumps({"summary": {"pandas": pandas.__version__, "numpy": numpy.__version__, + "pyarrow": pyarrow.__version__}}, sort_keys=True)) diff --git a/examples/pandas-asv/run.sh b/examples/pandas-asv/run.sh new file mode 100755 index 0000000..0902820 --- /dev/null +++ b/examples/pandas-asv/run.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +# One benchmark from pandas' own asv suite, run unchanged by `bx bench PATH --adapter asv` (asv's +# benchmark format, timed by pyperf) under two pandas versions, each in its own venv, alternated +# round by round, then compared. See README.md for what each step shows. +# +# Needs `uv`, git, and an active environment with benchx installed in editable mode from a +# benchx checkout (`pip install -e /prototype`). That checkout is installed in both +# venvs; set BENCHX_SRC to the directory with its pyproject.toml to name it instead. +# Everything it creates is here: pandas/, .venv-a, .venv-b and out/ (git-ignored). +# benchx builds nothing, and neither asv nor asv_runner is installed. +set -euo pipefail +cd "$(dirname "$0")" + +# Where the venvs get benchx from: the checkout that the active environment's benchx runs from. +BENCHX_SRC=${BENCHX_SRC:-} +if [ -z "$BENCHX_SRC" ]; then + BENCHX_SRC=$(python -c 'import benchx, pathlib; print(pathlib.Path(benchx.__file__).resolve().parents[1])' 2>/dev/null) || true +fi +if [ -z "$BENCHX_SRC" ] || [ ! -f "$BENCHX_SRC/pyproject.toml" ]; then + echo "run.sh: cannot find a benchx checkout to install in the venvs." >&2 + echo " Install benchx in editable mode in the active environment (pip install -e /prototype)," >&2 + echo " or set BENCHX_SRC to the directory with benchx's pyproject.toml." >&2 + exit 1 +fi + +# The suite is pandas' asv_bench at one tag, the same for both sides: only the installed pandas +# differs. The case is one parameter combination of one benchmark: `str` is object dtype in +# pandas 2.x, and the pyarrow-backed string dtype from pandas 3.0. +PANDAS_REF=v3.1.1 +PANDAS_A=2.3.3 +PANDAS_B=3.1.1 +CASE="strings.Methods.time_lower('str')" +PYTHON_VERSION=3.12 +ROUNDS=3 # the fewest that give a verdict +SIDES=("a $PANDAS_A" "b $PANDAS_B") +RUN_KEY=pandas-$(date +%s) +SLOT=0 + +echo "== 1. fetch pandas' asv suite at $PANDAS_REF (asv_bench/ only)" +if [ "$(git -C pandas describe --tags --exact-match 2>/dev/null)" != "$PANDAS_REF" ]; then + rm -rf pandas + git clone -q --depth 1 --branch "$PANDAS_REF" --filter=blob:none --sparse \ + https://github.com/pandas-dev/pandas.git pandas 2>/dev/null + git -C pandas sparse-checkout set asv_bench +fi +echo " pandas/asv_bench at $(git -C pandas rev-parse --short HEAD)" + +rm -rf out .venv-a .venv-b +mkdir -p out + +echo "== 2. the user prepares both environments; benchx builds nothing (benchx from $BENCHX_SRC)" +for SIDE_VERSION in "${SIDES[@]}"; do + read -r SIDE VERSION <<<"$SIDE_VERSION" + uv venv -q --python "$PYTHON_VERSION" ".venv-$SIDE" + uv pip install -q --python ".venv-$SIDE/bin/python" "pandas==$VERSION" pyarrow pyperf -e "$BENCHX_SRC" + echo " .venv-$SIDE: pandas $VERSION" +done + +echo "== 3. the caller's loop: alternate the sides, round by round, one slot per call" +for ROUND in $(seq 0 $((ROUNDS - 1))); do + for SIDE_VERSION in "${SIDES[@]}"; do + read -r SIDE VERSION <<<"$SIDE_VERSION" + # Activating a venv replaces the one activated before; no deactivate is needed. + source ".venv-$SIDE/bin/activate" + # Modules the venv cannot import (no matplotlib, scipy, ...) are listed on stderr and skipped. + bx bench pandas/asv_bench --adapter asv "$CASE" \ + --run-key "$RUN_KEY" --label "env=pandas-$VERSION" \ + --round "$ROUND" --slot "$SLOT" --out "out/$RUN_KEY" | tee "out/bench-$SLOT.txt" + echo " round $ROUND, slot $SLOT: side $SIDE" + SLOT=$((SLOT + 1)) + done +done + +echo "== 4. compare: the sides differ in the pandas the collector read" +bx compare --run "$RUN_KEY" --profile environments --label env \ + --baseline "pandas-$PANDAS_A" | tee out/compare.txt +grep -q 'differs:.*deps\.pandas' out/compare.txt # the differing fact is shown +if grep -q 'failed invariants' out/compare.txt; then echo "the comparison was refused" >&2; exit 1; fi + +echo "== 5. the series: one per environment, each point at the suite's revision" +bx series From 03274cc267bba06b5dc85cab93ba47611791d179 Mon Sep 17 00:00:00 2001 From: MarcoGorelli <33491632+MarcoGorelli@users.noreply.github.com> Date: Wed, 18 Nov 2026 16:35:20 +0000 Subject: [PATCH 2/2] pandas-asv: run with released benchx, keep the store local, explain the output --- examples/pandas-asv/README.md | 24 +++++++++++------- examples/pandas-asv/run.sh | 47 ++++++++++++++++++++++++----------- 2 files changed, 48 insertions(+), 23 deletions(-) diff --git a/examples/pandas-asv/README.md b/examples/pandas-asv/README.md index 3b076d9..6151479 100644 --- a/examples/pandas-asv/README.md +++ b/examples/pandas-asv/README.md @@ -17,19 +17,17 @@ so the two sides are expected to differ. ## Run it -You need [`uv`](https://docs.astral.sh/uv/), git, and network access the first time. You also -need benchx, installed in editable mode from a checkout, in the active environment. Then, from +You need [`uv`](https://docs.astral.sh/uv/), git, and network access the first time. Then, from this directory: ```console -$ git clone https://github.com/benchx-dev/benchx.git -$ python3 -m venv .venv && . .venv/bin/activate # Python >= 3.11, or any environment you like -$ pip install -e benchx/prototype $ ./run.sh ``` -The script installs that same checkout in both venvs. It finds it from the benchx in the active -environment; `BENCHX_SRC=/prototype ./run.sh` names it instead. +benchx itself runs only inside the two venvs, and the script installs it there: the +benchx of the active environment if there is one (its checkout if it is an editable install, +else the same version from PyPI), else the latest release from PyPI. To try a benchx checkout +without installing it anywhere, name it: `BENCHX_SRC=path/to/benchx ./run.sh`. The tag, the two pandas versions, the case, the Python and the number of rounds are set at the top of `run.sh`. Everything it creates is in `pandas/`, `.venv-a`, `.venv-b` and `out/`, all @@ -71,7 +69,13 @@ git-ignored. A run takes under a minute after the venvs are cached. strings.Methods.time_lower wall-time effect -85.40% noise 1.76% rounds=3 ``` - (The numbers vary from run to run.) + (The numbers vary from run to run.) `effect` is the contender against the baseline, averaged + over the rounds' paired medians: `-85.40%` means pandas 3.1.1 takes about 15% of the time + pandas 2.3.3 does, so about 7x faster, well beyond the round-to-round `noise`. `differs` + names the environment facts that tell the sides apart: `deps.pandas` is the version the + collector printed, and `deps.sha256` is the hash of the collector's whole output, which + changes whenever any of its facts do. Here numpy and pyarrow are the same on both sides, so + pandas is the one difference. 5. **Lists the series.** One per environment, because the collected pandas version is part of a series' identity. Every point carries the suite's revision (pandas `v3.1.1`), whichever pandas it ran under: the target is the checkout the benchmarks come from, and what varies is @@ -90,7 +94,9 @@ git-ignored. A run takes under a minute after the venvs are cached. in the environment the call runs in, and it prints the pandas, numpy and pyarrow it imports. That lands in each result's `environment.system.deps`, which is how the `environments` comparison knows what differs between the sides. -- `paths`: the store and the results go in `out/`. +- `paths`: the store and the results go in `out/`, which `run.sh` clears first, so `bx series` + shows this run only. `$BENCHX_HOME` would relocate the store over this setting, so `run.sh` + unsets it. Each result also records the installed project's version (`provenance.info.asv.installed`) and asv's hash of the benchmark's code (`protocol.asv.benchmark_version`), so editing the benchmark diff --git a/examples/pandas-asv/run.sh b/examples/pandas-asv/run.sh index 0902820..9c24357 100755 --- a/examples/pandas-asv/run.sh +++ b/examples/pandas-asv/run.sh @@ -3,26 +3,43 @@ # benchmark format, timed by pyperf) under two pandas versions, each in its own venv, alternated # round by round, then compared. See README.md for what each step shows. # -# Needs `uv`, git, and an active environment with benchx installed in editable mode from a -# benchx checkout (`pip install -e /prototype`). That checkout is installed in both -# venvs; set BENCHX_SRC to the directory with its pyproject.toml to name it instead. +# Needs `uv` and git. Both venvs get the benchx of the active environment: its checkout if it is +# an editable install, else the same version from PyPI (the latest if there is none). Set +# BENCHX_SRC to a benchx checkout (the directory with its pyproject.toml) to install that instead. # Everything it creates is here: pandas/, .venv-a, .venv-b and out/ (git-ignored). # benchx builds nothing, and neither asv nor asv_runner is installed. set -euo pipefail cd "$(dirname "$0")" -# Where the venvs get benchx from: the checkout that the active environment's benchx runs from. +# Where the venvs get benchx from: BENCHX_SRC, else the checkout of an editable benchx in the +# active environment, else PyPI, at the active environment's version if it has one. BENCHX_SRC=${BENCHX_SRC:-} if [ -z "$BENCHX_SRC" ]; then - BENCHX_SRC=$(python -c 'import benchx, pathlib; print(pathlib.Path(benchx.__file__).resolve().parents[1])' 2>/dev/null) || true + BENCHX_SRC=$(python - 2>/dev/null <<'EOF' || true +import importlib.metadata as md, json, urllib.parse +url = {} +for name in ("benchx", "benchx-prototype"): # older checkouts install benchx-prototype + try: + url = json.loads(md.distribution(name).read_text("direct_url.json") or "{}") + break + except md.PackageNotFoundError: + pass +if url.get("dir_info", {}).get("editable"): + print(urllib.parse.unquote(urllib.parse.urlparse(url["url"]).path)) +EOF +) fi -if [ -z "$BENCHX_SRC" ] || [ ! -f "$BENCHX_SRC/pyproject.toml" ]; then - echo "run.sh: cannot find a benchx checkout to install in the venvs." >&2 - echo " Install benchx in editable mode in the active environment (pip install -e /prototype)," >&2 - echo " or set BENCHX_SRC to the directory with benchx's pyproject.toml." >&2 - exit 1 +if [ -n "$BENCHX_SRC" ]; then + BENCHX_REQ=(-e "$BENCHX_SRC") +else + BENCHX_VERSION=$(python -c 'import importlib.metadata as m; print(m.version("benchx"))' 2>/dev/null || true) + BENCHX_REQ=("benchx${BENCHX_VERSION:+==$BENCHX_VERSION}") fi +# The store is out/store.parquet (benchx.conf.json). $BENCHX_HOME would relocate it, and mix +# this run's series with whatever else is stored there. +unset BENCHX_HOME + # The suite is pandas' asv_bench at one tag, the same for both sides: only the installed pandas # differs. The case is one parameter combination of one benchmark: `str` is object dtype in # pandas 2.x, and the pyarrow-backed string dtype from pandas 3.0. @@ -48,11 +65,11 @@ echo " pandas/asv_bench at $(git -C pandas rev-parse --short HEAD)" rm -rf out .venv-a .venv-b mkdir -p out -echo "== 2. the user prepares both environments; benchx builds nothing (benchx from $BENCHX_SRC)" +echo "== 2. the user prepares both environments; benchx builds nothing (benchx: ${BENCHX_REQ[*]})" for SIDE_VERSION in "${SIDES[@]}"; do read -r SIDE VERSION <<<"$SIDE_VERSION" uv venv -q --python "$PYTHON_VERSION" ".venv-$SIDE" - uv pip install -q --python ".venv-$SIDE/bin/python" "pandas==$VERSION" pyarrow pyperf -e "$BENCHX_SRC" + uv pip install -q --python ".venv-$SIDE/bin/python" "pandas==$VERSION" pyarrow pyperf "${BENCHX_REQ[@]}" echo " .venv-$SIDE: pandas $VERSION" done @@ -74,8 +91,10 @@ done echo "== 4. compare: the sides differ in the pandas the collector read" bx compare --run "$RUN_KEY" --profile environments --label env \ --baseline "pandas-$PANDAS_A" | tee out/compare.txt -grep -q 'differs:.*deps\.pandas' out/compare.txt # the differing fact is shown -if grep -q 'failed invariants' out/compare.txt; then echo "the comparison was refused" >&2; exit 1; fi +if grep -q 'failed invariants' out/compare.txt; then echo "run.sh: the comparison was refused" >&2; exit 1; fi +if ! grep -q 'differs:.*deps\.pandas' out/compare.txt; then # the differing fact is shown + echo "run.sh: the comparison does not name deps.pandas as differing" >&2; exit 1 +fi echo "== 5. the series: one per environment, each point at the suite's revision" bx series