diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 0f09a552..714c5966 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -102,7 +102,7 @@ jobs: working-directory: benchx run: | uv python install 3.12 - uv sync --extra test --python 3.12 + uv sync --extra parquet --extra test --python 3.12 echo "$PWD/.venv/bin" >> "$GITHUB_PATH" - name: Install NumPy's build tools @@ -208,7 +208,7 @@ jobs: working-directory: benchx run: | uv python install 3.13 - uv sync --extra test --python 3.13 + uv sync --extra parquet --extra test --python 3.13 echo "$PWD/.venv/bin" >> "$GITHUB_PATH" # cmake and a C++ compiler are preinstalled on the runner; CMake fetches @@ -270,7 +270,7 @@ jobs: working-directory: benchx run: | uv python install 3.13 - uv sync --extra test --python 3.13 + uv sync --extra parquet --extra test --python 3.13 echo "$PWD/.venv/bin" >> "$GITHUB_PATH" - name: Show what is under test @@ -325,7 +325,7 @@ jobs: working-directory: benchx run: | uv python install 3.13 - uv sync --extra test --python 3.13 + uv sync --extra parquet --extra test --python 3.13 echo "$PWD/.venv/bin" >> "$GITHUB_PATH" # The README's install lines. torch comes from PyTorch's CPU index: the default diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 67716744..dd851875 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -72,7 +72,7 @@ jobs: run: | uv venv --python 3.13 "$RUNNER_TEMP/venv" source "$RUNNER_TEMP/venv/bin/activate" - uv pip install "$(ls dist/*.whl)[test]" + uv pip install "$(ls dist/*.whl)[parquet,test]" # Away from the checkout, so that nothing can import it instead of the installed package. cd "$RUNNER_TEMP" bx --help diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 4369d9d0..af7821b0 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -46,10 +46,40 @@ jobs: run: uv python install ${{ matrix.python-version }} - name: Install package and test dependencies - run: uv sync --extra test --python ${{ matrix.python-version }} + run: uv sync --extra parquet --extra test --python ${{ matrix.python-version }} - name: Run tests - run: uv run --extra test --python ${{ matrix.python-version }} python -m pytest tests -ra + run: uv run --extra parquet --extra test --python ${{ matrix.python-version }} python -m pytest tests -ra + + minimal: + name: minimal (no pyarrow) / py3.13 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: astral-sh/setup-uv@v5 + with: + enable-cache: true + cache-dependency-glob: uv.lock + + - name: Install Python 3.13 + run: uv python install 3.13 + + - name: Install package without parquet extra + run: uv sync --extra test --python 3.13 + + - name: Verify pyarrow and jsonpointer are absent + run: | + uv run --extra test --python 3.13 python -c ' + import importlib.util + for mod in ("pyarrow", "jsonpointer"): + assert importlib.util.find_spec(mod) is None, f"unexpected dependency installed: {mod}" + ' + + - name: Run tests without pyarrow + run: | + uv run --extra test --python 3.13 bx --help > /dev/null + uv run --extra test --python 3.13 python -m pytest tests -ra # An editable install finds the schemas next to the source, so the matrix above cannot # tell whether a wheel carries them. This installs the built wheel (not editable) into a @@ -70,8 +100,20 @@ jobs: run: | uv venv --python 3.13 "$RUNNER_TEMP/venv" source "$RUNNER_TEMP/venv/bin/activate" - uv pip install "$(ls dist/*.whl)[test]" + uv pip install "$(ls dist/*.whl)" # Away from the checkout, so that nothing can import it instead of the installed package. cd "$RUNNER_TEMP" + python -c ' + import importlib.metadata + reqs = importlib.metadata.requires("benchx") or [] + default_reqs = [r for r in reqs if "extra ==" not in r] + parquet_reqs = [r for r in reqs if "extra ==" in r and "parquet" in r] + assert any(r.startswith("pyarrow") for r in parquet_reqs), reqs + for unwanted in ("pyarrow", "jsonpointer", "format-nongpl", "fqdn", "idna", "isoduration", "rfc3987-syntax", "uri-template", "webcolors"): + assert not any(unwanted in r for r in default_reqs), f"unexpected default dep {unwanted!r} in {default_reqs}" + ' bx --help + uv pip install "$(ls "$GITHUB_WORKSPACE"/dist/*.whl)[test]" + python -m pytest --pyargs benchx -ra + uv pip install "$(ls "$GITHUB_WORKSPACE"/dist/*.whl)[parquet,test]" python -m pytest --pyargs benchx -ra diff --git a/README.md b/README.md index c3cea466..ff702553 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ allow all of them. ## BenchX usage and development -Install with `pip install benchx` ([PyPI](https://pypi.org/project/benchx/)). +Install with `pip install benchx` (or `pip install 'benchx[parquet]'` for the local Parquet store) ([PyPI](https://pypi.org/project/benchx/)). We welcome contributions of all sorts: add a user story, tell us about your use case, try `benchx` on your workflow/project, open an issue if you found a rough edge, diff --git a/docs/benchx.md b/docs/benchx.md index 55436065..7281a2be 100644 --- a/docs/benchx.md +++ b/docs/benchx.md @@ -24,7 +24,8 @@ implement a missing feature. Install from [PyPI](https://pypi.org/project/benchx/): ```console -$ pip install benchx +$ pip install benchx # minimal: run with --no-ingest, validate, and compare --results +$ pip install 'benchx[parquet]' # adds PyArrow for the local Parquet store and history queries ``` @@ -55,7 +56,7 @@ bx compare BASELINE CONTENDER --profile revisions|environments --suite NAME [--filter REGEX] [--quantity wall-time] --rounds R [-e NAME=VALUE ...] [--repetitions N] [--min-time S] [--project P] [--label NAME=BASE,CONTENDER] - [--run-key KEY] [--out DIR] [--no-build] [--k 3] [--json | --tables [--all]] + [--run-key KEY] [--out DIR] [--no-build] [--no-ingest] [--k 3] [--json | --tables [--all]] # measure mode, one target: alternate the two values of a benchmark parameter bx compare TARGET --profile variants --adapter pyperf|asv --suite NAME --rounds R @@ -73,6 +74,10 @@ bx ingest # result files and work orders → the bx series [--workload … --quantity …] # list series bx history # one series' points (a query); dirty ones are marked bx head [-n 10] [--project P] [--workload W] [--json|--table] # the latest results, newest first; dirty ones are marked +bx view [--out DIR] [--serve] [--host 127.0.0.1] [--port 8000] [--open] + [--store PATH | --results PATH… | --runs DIR] + [--project-file PATH] [--title TITLE] [--repo DIR] [--marts | --no-marts] [--stamp STR] + # build and serve the static benchmark dashboard (no --out: temp dir + serve) bx target describe|prepare TARGET # a target's description (a provider's answer, or a build directory's sidecar) bx validate … [--kind KIND] # every problem in each document, by schema and rules @@ -247,7 +252,8 @@ outside the checkout is identified by one file): put it inside the checkout. and it refuses an order that names a deferred field (`environment_policy`, a declared `build`, `components`, `schedule`) instead of ignoring it. - **The store** is one Parquet file, by default `store.parquet` in the project root - (`paths.store` in `benchx.conf.json` moves it). It reads documents strictly, validates them, and applies the + (`paths.store` in `benchx.conf.json` moves it), and requires the optional `parquet` extra + (`pip install 'benchx[parquet]'`). It reads documents strictly, validates them, and applies the schema §5.4 contract: idempotency and the rejection codes `malformed`, `idempotency-conflict`, `unit-conflict`, `identity-violation`, and `attempt-conflict`. Each result is one row: the columns @@ -257,6 +263,12 @@ outside the checkout is identified by one file): put it inside the checkout. query, under the identity policy in `benchx/identity.py`: the default of `schemas/identity-policy/0.1.0` with the benchmark's revision ignored. Every accepting ingest rewrites the file atomically. +- **Schema validation** checks documents against the schemas shipped inside the package, or custom + schemas pointed to by `$BENCHX_SCHEMAS`. The shipped schemas use only the `date-time` and `uri` + JSON Schema formats, checked via `rfc3339-validator` and `rfc3986-validator`. Custom schemas + supplied through `$BENCHX_SCHEMAS` that declare additional formats require the corresponding + `jsonschema` format validator package (or `jsonschema[format-nongpl]`); if a declared format has + no installed checker, validation fails explicitly rather than silently accepting unchecked values. - **The session loop** (`benchx/session.py`, `bx compare` in measure mode) is the workbench's part: it checks both targets (and, for `revisions`, that both trees have a working-tree id, before running anything; it says which sides @@ -278,8 +290,8 @@ outside the checkout is identified by one file): put it inside the checkout. |effect| > *k* × noise). `environments` names its sides by the label `--label` gives, and `variants` by the two values of one workload parameter (`--vary`); under `variants` a failed attempt on either side fails its unit - (`side-failed`). `--results` compares result files through a throwaway - store: the ad hoc path. Condition measurements (`measurement.kind: + (`side-failed`). `--results` validates and compares JSON result files in memory + without PyArrow or a Parquet store: the ad hoc path. Condition measurements (`measurement.kind: condition`) are not compared, and deterministic quantities are deferred. - **The tables** (`--tables`, `benchx/tables.py`) print, after the usual summary, what `asv continuous` prints, laid out and rounded as asv does: a table of each side (a value @@ -290,12 +302,20 @@ outside the checkout is identified by one file): put it inside the checkout. deviation (not asv's `±`); `Change` and `Verdict` are the comparator's verdict (`+` regressed, `-` improved), `Noise` the one it compares the effect with (a change is more than *k* times it), and `Ratio` is After over Before. A text view: not with `--json`. +- **Views** (`bx view`, `benchx/views/`) derive flat tables (`commits`, `points`, `changes`, + `latest`, `movers`) and static JSON view payloads (`landing`, `series`, `sparks`, + `history/`, `commits/`) in pure Python from the local Parquet store, JSON result + files (`--results`, no PyArrow required), or a CI `runs///` directory (`--runs`, + with `run.parquet` and `results.parquet`). With no `--out`, `bx view` builds into a temporary + directory and serves the bundled static dashboard over local HTTP until Ctrl-C; with `--out DIR`, + it writes a deployable static site (and optional Parquet marts under `data/` when `pyarrow` is + installed, unless `--no-marts`). ## Run it ```console -$ uv venv && uv pip install -e '.[test]' # editable; a plain `uv pip install .` works too, the schemas ship in the package -$ .venv/bin/python -m pytest tests # 470 tests, fake Google Benchmark binary, no compiler +$ uv venv && uv pip install -e '.[parquet,test]' # editable; omit `parquet` for a minimal Arrow-free install +$ .venv/bin/python -m pytest tests # fake Google Benchmark binary, no compiler ``` `tests/test_e2e_numpy_revisions.py` is an end-to-end test that builds NumPy with @@ -308,7 +328,8 @@ keeps the builds between runs, `BENCHX_E2E_REVISIONS` picks the two tags, scenario, and the test's docstring has the rest. A regular (non-editable) install ships the test suite too, so an installed copy can test itself: -`pip install 'benchx[test]'` followed by `python -m pytest --pyargs benchx`. +`pip install 'benchx[parquet,test]'` (or `pip install 'benchx[test]'` for the Arrow-free subset) +followed by `python -m pytest --pyargs benchx`. The end-to-end examples that build real software are in [benchx_examples](https://github.com/benchx-dev/benchx_examples), and the `e2e` diff --git a/docs/releasing.md b/docs/releasing.md index c1e1deae..68b14997 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -32,7 +32,7 @@ package, waits for approval, publishes to PyPI, and creates the GitHub release. ```console $ python -m venv venv && source venv/bin/activate - $ pip install 'benchx[test]==0.1.0' + $ pip install 'benchx[parquet,test]==0.1.0' $ bx --help $ python -m pytest --pyargs benchx ``` @@ -42,8 +42,8 @@ package, waits for approval, publishes to PyPI, and creates the GitHub release. After a new version reaches PyPI, conda-forge's bot opens a pull request there, which can take several hours. A maintainer of the feedstock then: - Compares the recipe's dependencies with `dependencies` in `pyproject.toml`, and updates the - pull request if they changed. `jsonschema[format-nongpl]` is called - `jsonschema-with-format-nongpl` on conda-forge. + pull request if they changed (`pyarrow` is optional via the `parquet` extra, and format + validation uses `rfc3339-validator` and `rfc3986-validator` directly). - Merges the pull request once its CI passes. - Tests the install, once the package is available: diff --git a/pyproject.toml b/pyproject.toml index 0e766695..1cd4e6d8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,10 +6,10 @@ license = "Apache-2.0" license-files = ["LICENSE"] requires-python = ">=3.11" dependencies = [ - "pyarrow>=19", - # format-nongpl makes `format: uri` and `format: date-time` actually checked. - "jsonschema[format-nongpl]>=4.18", - "jsonpointer>=2.0", + "jsonschema>=4.18", + # Format validators for the formats used by the shipped schemas (`date-time`, `uri`). + "rfc3339-validator", + "rfc3986-validator>0.1.0", "referencing", "rfc8785", ] @@ -24,6 +24,7 @@ Documentation = "https://benchx-dev.github.io/benchx/benchx/" Repository = "https://github.com/benchx-dev/benchx" [project.optional-dependencies] +parquet = ["pyarrow>=19"] test = ["pytest", "pyperf", "asv_runner"] [project.scripts] diff --git a/src/benchx/bench.py b/src/benchx/bench.py index d4c1c795..cccab340 100644 --- a/src/benchx/bench.py +++ b/src/benchx/bench.py @@ -15,7 +15,7 @@ from . import core, snapshot from .errors import SessionError from .projection import declared_workload, said, workload_label -from .store import warning_codes +from .results import warning_codes def tree_of(module: Path, tree=None) -> Path: diff --git a/src/benchx/cli.py b/src/benchx/cli.py index 07e155e8..7a0313e5 100644 --- a/src/benchx/cli.py +++ b/src/benchx/cli.py @@ -20,8 +20,9 @@ from . import core, harnesses, orders, runner, session, snapshot from . import tables as tables_mod from . import target as target_mod +from . import views as views_mod from .core.validation import Kind -from .store import Store, warning_codes +from .results import MemoryStore, warning_codes EXIT_OK = 0 EXIT_INVALID = 1 # a document invalid, a result rejected, or a target without a description @@ -47,12 +48,34 @@ def _load(command): return cfg +def _open_store(path, command: str, *, hint: str | None = None): + """Open the local Parquet store, or explain how to install pyarrow and return None.""" + try: + from .store import Store + except ModuleNotFoundError as exc: + if exc.name and exc.name.split(".")[0] == "pyarrow": + suffix = f", or {hint}" if hint else "" + print( + f"bx {command}: error: Parquet store requires pyarrow; " + f"install it with `pip install 'benchx[parquet]'`{suffix}", + file=sys.stderr, + ) + return None + raise + return Store(path) + + def cmd_run(args): """Run one order, write its result files, and deliver them to the local store. The runner itself stays store-unaware (runner.md §2 principle 6); this command composes it with ingest, as the workbench does.""" if (cfg := _load("run")) is None: return EXIT_REFUSED + store = None + if not args.no_ingest: + store = _open_store(cfg.store_path, "run", hint="pass --no-ingest to write JSON results only") + if store is None: + return EXIT_REFUSED out = Path(args.out) if args.out else cfg.results_dir snapshot.ignore(cfg.store_path, cfg.results_dir, out) try: @@ -61,9 +84,8 @@ def cmd_run(args): return _refused(e) print(f"{summary['results']} result(s) for {summary['cases']} case(s) -> {summary['out']}" f" (order {summary['order'][:19]})") - if args.no_ingest: + if store is None: return EXIT_OK - store = Store(cfg.store_path) print(f"{store.path}: ", end="") return _report(store.ingest_paths(summary["files"])) @@ -143,10 +165,12 @@ def _common(cfg, args, *, forced_adapter=None, forced_suite=None) -> dict: raise UsageError("give --project or --no-project, not both") timeouts = cfg.section("timeouts") adapter = cfg.pick(given, block.get("name"), flag="--adapter", key="adapter.name") + flags = {name: getattr(args, name) for name in harnesses.FLAG_NAMES} + harnesses.get(adapter).flags_to_precision(**flags) return { "adapter": adapter, "suite": cfg.pick(forced_suite or args.suite, block.get("suite"), flag="--suite", key="adapter.suite"), - "flags": {name: getattr(args, name) for name in harnesses.FLAG_NAMES}, + "flags": flags, "file_precision": block.get("precision"), "quantities": quantities, "filter": args.filter, @@ -195,7 +219,11 @@ def cmd_bench(args): return EXIT_REFUSED settings["filter"] = names or settings["filter"] snapshot.ignore(cfg.store_path, cfg.results_dir, *([args.out] if args.out else [])) - store = None if args.no_ingest else Store(cfg.store_path) + store = None + if not args.no_ingest: + store = _open_store(cfg.store_path, "bench", hint="pass --no-ingest to write JSON results only") + if store is None: + return EXIT_REFUSED try: summary = session.bench_target(target, store=store, progress=lambda message: print(message, file=sys.stderr), **settings) @@ -276,13 +304,17 @@ def _report(outcome): def cmd_ingest(args): if (cfg := _load("ingest")) is None: return EXIT_REFUSED - return _report(Store(cfg.store_path).ingest_paths(args.paths)) + if (store := _open_store(cfg.store_path, "ingest")) is None: + return EXIT_REFUSED + return _report(store.ingest_paths(args.paths)) def cmd_series(args): if (cfg := _load("series")) is None: return EXIT_REFUSED - for s in Store(cfg.store_path).series(args.workload, args.quantity): + if (store := _open_store(cfg.store_path, "series")) is None: + return EXIT_REFUSED + for s in store.series(args.workload, args.quantity): estimator = json.loads(s["estimator"])["method_version"] print(f"{s['fingerprint'][:12]} {s['project']} {s['workload']} {s['quantity']} [{s['unit']}]" f" est={estimator}") @@ -292,7 +324,9 @@ def cmd_series(args): def cmd_history(args): if (cfg := _load("history")) is None: return EXIT_REFUSED - points = Store(cfg.store_path).history(args.series) + if (store := _open_store(cfg.store_path, "history")) is None: + return EXIT_REFUSED + points = store.history(args.series) print("revision order unavailable without a revision graph; listed by measurement start") for p in points: changed = f" changed: {', '.join(p['changed'])}" if p["changed"] else "" @@ -305,7 +339,8 @@ def cmd_history(args): def cmd_head(args): if (cfg := _load("head")) is None: return EXIT_REFUSED - store = Store(cfg.store_path) + if (store := _open_store(cfg.store_path, "head")) is None: + return EXIT_REFUSED total, rows = store.head(args.n, args.project, args.workload, measurands=args.table) if args.json: for row in rows: @@ -356,7 +391,7 @@ class UsageError(Exception): # Flags that belong to one mode of `bx compare` (prototype-design.md §1). _MEASURE_ONLY = ("adapter", "suite", "filter", "quantity", "rounds", "env", *harnesses.FLAG_NAMES, "project", "no_project", - "run_key", "out", "source_uri", "no_build") + "run_key", "out", "source_uri", "no_build", "no_ingest") _READ_ONLY = ("baseline", "results") @@ -432,31 +467,49 @@ def _view_flags(args): def _read(cfg, args, policy): if args.results: - with tempfile.TemporaryDirectory() as tmp: # the ad hoc path: a throwaway store - store = Store(Path(tmp) / "store.parquet") - outcome = store.ingest_paths(args.results) - if outcome["rejected"]: - return _report(outcome) - return _compare(store, args, policy) - return _compare(Store(cfg.store_path), args, policy) + store = MemoryStore() # the ad hoc path: validated in memory without PyArrow + outcome = store.ingest_paths(args.results) + if outcome["rejected"]: + return _report(outcome) + return _compare(store, args, policy) + store = _open_store(cfg.store_path, "compare", hint="pass --results DIR to compare JSON results directly") + if store is None: + return EXIT_REFUSED + return _compare(store, args, policy) def _measure(cfg, args, policy): """The workbench's session loop: run both sides, deliver, compare.""" - store = Store(cfg.store_path) - print(f"{store.path}", file=sys.stderr) # stderr, so --json output stays clean try: settings = _common(cfg, args) _needs_provider(cfg, *args.targets) - snapshot.ignore(cfg.store_path, cfg.results_dir, *([args.out] if args.out else [])) rounds = cfg.pick(args.rounds, cfg.section("compare").get("rounds"), flag="--rounds", key="compare.rounds") + label = session.parse_label(args.label) if policy["profile"] != "variants" and args.label else None + except (UsageError, config_mod.ConfigError) as e: + print(f"bx compare: error: {e}", file=sys.stderr) + return EXIT_REFUSED + except session.SessionError as e: + print(f"cannot compare: {e}", file=sys.stderr) + return EXIT_REFUSED + if args.no_ingest: + store = MemoryStore() + else: + store = _open_store( + cfg.store_path, + "compare", + hint="pass --no-ingest to compare in memory without persisting to the local store", + ) + if store is None: + return EXIT_REFUSED + print(f"{store.path}", file=sys.stderr) # stderr, so --json output stays clean + try: + snapshot.ignore(cfg.store_path, cfg.results_dir, *([args.out] if args.out else [])) progress = lambda message: print(message, file=sys.stderr) # noqa: E731 if policy["profile"] == "variants": doc = session.compare_variants( *args.targets, rounds=rounds, store=store, parameter=args.vary, baseline=args.baseline, progress=progress, k=policy["k"], min_rounds=policy["min_rounds"], **settings) else: - label = session.parse_label(args.label) if args.label else None doc = session.compare_targets(*args.targets, rounds=rounds, store=store, label=label, progress=progress, **policy, **settings) except runner.Refused as e: @@ -510,6 +563,105 @@ def cmd_validate(args): return EXIT_OK if valid == len(args.files) else EXIT_INVALID +def cmd_view(args): + """Build static dashboard views and optionally serve them over local HTTP.""" + if (cfg := _load("view")) is None: + return EXIT_REFUSED + serve = args.serve or args.out is None + if args.open and not serve: + print("bx view: error: --open goes with serving (omit --out, or pass --serve)", file=sys.stderr) + return EXIT_REFUSED + + source_kwargs: dict = {} + if args.runs is not None: + source_kwargs["runs_dir"] = args.runs + elif args.results is not None: + source_kwargs["results_paths"] = list(args.results) + else: + source_kwargs["store_path"] = args.store if args.store is not None else cfg.store_path + + if args.marts: + write_marts: bool | None = True + elif args.no_marts: + write_marts = False + elif args.out is None: + write_marts = False + else: + write_marts = None + + repo = Path(args.repo) if args.repo is not None else cfg.root + build_kwargs = { + **source_kwargs, + "project_file": Path(args.project_file) if args.project_file else None, + "title": args.title, + "repo": repo, + "write_marts": write_marts, + "stamp": args.stamp, + "config": cfg, + } + + def _run_build(out_dir: Path) -> dict | None: + try: + return views_mod.build_site(out_dir, **build_kwargs) + except ModuleNotFoundError as exc: + if exc.name and exc.name.split(".")[0] == "pyarrow": + hint = "" + if "store_path" in source_kwargs: + hint = ", or pass --results DIR to build views from JSON results directly" + print(f"bx view: error: {exc}{hint}", file=sys.stderr) + return None + raise + except views_mod.ViewError as exc: + print(f"bx view: error: {exc}", file=sys.stderr) + return None + + port = 8000 if args.port is None else args.port + fallback_port = args.port is None + + def _finish(summary: dict, out_dir: Path) -> int: + exit_code = EXIT_OK + if summary["outcome"] and summary["outcome"]["rejected"]: + exit_code = _report(summary["outcome"]) + counts = summary["counts"] + if args.out is not None: + print( + f"{counts['points']} result(s), {counts['series']} series, " + f"{counts['distinct_commits']} commit(s) -> {summary['out']}" + ) + else: + print( + f"{counts['points']} result(s), {counts['series']} series, " + f"{counts['distinct_commits']} commit(s)" + ) + if serve: + try: + views_mod.serve_site( + out_dir, + host=args.host, + port=port, + fallback_port=fallback_port, + open_browser=args.open, + ) + except OSError as exc: + print(f"bx view: error: cannot serve on {args.host}:{port}: {exc}", file=sys.stderr) + return EXIT_REFUSED + return exit_code + + if args.out is not None: + out_dir = Path(args.out) + summary = _run_build(out_dir) + if summary is None: + return EXIT_REFUSED + return _finish(summary, out_dir) + + with tempfile.TemporaryDirectory(prefix="benchx-view-") as tmp: + out_dir = Path(tmp) + summary = _run_build(out_dir) + if summary is None: + return EXIT_REFUSED + return _finish(summary, out_dir) + + def _add_target_options(parser, command): """The options `bx bench` and `bx compare` (measure mode) share: the harness, its suite and cases, its precision flags, and where the results go.""" @@ -666,10 +818,12 @@ def main(argv=None): read = p.add_argument_group("read mode") read.add_argument("--run", help="run key") read.add_argument("--results", nargs="+", - help="compare these result directories through a throwaway store, not the local one") + help="compare these result files or directories directly, without using the local store") measure = p.add_argument_group("measure mode") measure.add_argument("--rounds", type=int, help="alternation rounds; each runs both sides once (default: compare.rounds)") + measure.add_argument("--no-ingest", action="store_true", + help="write result files and compare in memory; skip the local store") _add_target_options(measure, "compare") p.set_defaults(fn=cmd_compare) @@ -680,6 +834,36 @@ def main(argv=None): help="kind of document being checked (default: %(default)s)") p.set_defaults(fn=cmd_validate) + p = sub.add_parser( + "view", + help="build and serve the static benchmark dashboard", + description="Build static dashboard views from the local Parquet store, JSON result files " + "(--results, no PyArrow required), or a runs/// directory (--runs). " + "With no --out, builds into a temporary directory and serves locally until Ctrl-C; " + "with --out DIR, writes the static site (add --serve to also serve it).", + ) + src = p.add_mutually_exclusive_group() + src.add_argument("--store", help="local Parquet store (default: paths.store / $BENCHX_HOME/store.parquet)") + src.add_argument("--results", nargs="+", + help="JSON result files or directories, read directly without PyArrow") + src.add_argument("--runs", help="runs/// directory holding run.parquet and results.parquet") + p.add_argument("--out", help="write the static site into DIR instead of a temporary directory") + p.add_argument("--serve", action="store_true", help="serve the built site over local HTTP") + p.add_argument("--host", default="127.0.0.1", help="bind address for local serving (default %(default)s)") + p.add_argument("--port", type=int, default=None, + help="TCP port for local serving (default 8000, falling back to a free port if busy)") + p.add_argument("--open", action="store_true", help="open the served dashboard in the default web browser") + p.add_argument("--project-file", help="project.json describing title, links, and upstream commit URL") + p.add_argument("--title", help="override the dashboard page title") + p.add_argument("--repo", help="git checkout used to enrich commit metadata for --store / --results") + marts_group = p.add_mutually_exclusive_group() + marts_group.add_argument("--marts", action="store_true", + help="write derived Parquet tables under data/ (requires benchx[parquet])") + marts_group.add_argument("--no-marts", action="store_true", + help="skip writing derived Parquet tables under data/") + p.add_argument("--stamp", help="cache-busting version stamped onto static asset URLs in index.html") + p.set_defaults(fn=cmd_view) + args, extra = ap.parse_known_args(argv) # argparse takes a nargs="*" positional only from the first run of positionals, so cases # given after an option (`bx bench T --suite S CASE`) arrive here, on some Python versions. @@ -687,7 +871,11 @@ def main(argv=None): args.functions += extra elif extra: ap.error(f"unrecognized arguments: {' '.join(extra)}") - return args.fn(args) + try: + return args.fn(args) + except core.catalog.FormatCheckerError as e: + print(f"bx {args.cmd}: error: {e}", file=sys.stderr) + return EXIT_REFUSED if __name__ == "__main__": diff --git a/src/benchx/compare.py b/src/benchx/compare.py index 4427bd4b..033371a2 100644 --- a/src/benchx/compare.py +++ b/src/benchx/compare.py @@ -18,8 +18,8 @@ from . import identity from .core import canonical, catalog from .projection import Projection, axis_order, declared_workload, get, said, workload_label +from .results import store_project from .snapshot import DIRTY_WARNINGS -from .store import store_project METHOD = {"name": "benchx/paired-relative", "version": "v1", "effect_measure": "relative"} PROFILES = { diff --git a/src/benchx/core/catalog.py b/src/benchx/core/catalog.py index b0064c50..ac6dd730 100644 --- a/src/benchx/core/catalog.py +++ b/src/benchx/core/catalog.py @@ -1,4 +1,11 @@ -"""The schemas shipped inside the package (benchx/schemas), or in $BENCHX_SCHEMAS.""" +"""The schemas shipped inside the package (benchx/schemas), or in $BENCHX_SCHEMAS. + +Shipped schemas use only the `date-time` and `uri` JSON Schema formats, checked +via `rfc3339-validator` and `rfc3986-validator`. Custom schemas supplied through +`$BENCHX_SCHEMAS` may use additional formats if the corresponding validator +package (or `jsonschema[format-nongpl]`) is installed; missing format checkers +raise `FormatCheckerError` rather than silently skipping format validation. +""" import json import os @@ -24,6 +31,32 @@ RELEASE = re.compile(r"[0-9]+\.[0-9]+\.[0-9]+") +_SUBSCHEMA_KEYS = ( + "not", + "if", + "then", + "else", + "items", + "additionalProperties", + "contains", + "propertyNames", + "unevaluatedItems", + "unevaluatedProperties", + "contentSchema", +) +_SUBSCHEMA_LIST_KEYS = ("allOf", "anyOf", "oneOf", "prefixItems") +_SUBSCHEMA_MAP_KEYS = ( + "$defs", + "definitions", + "properties", + "patternProperties", + "dependentSchemas", +) + + +class FormatCheckerError(RuntimeError): + """A schema declares a `format` whose jsonschema checker is not installed.""" + def release(kind: str, version: str) -> str | None: """The release a declared version names, such as "0.1.0" for @@ -36,6 +69,57 @@ def release(kind: str, version: str) -> str | None: return None +def _required_formats(schema: dict, by_id: dict[str, dict]) -> set[str]: + """Every `format` keyword declared by `schema` and any catalog schemas it `$ref`s.""" + formats: set[str] = set() + visited_ids: set[str] = set() + if isinstance(schema.get("$id"), str): + visited_ids.add(schema["$id"]) + + def walk(node: object) -> None: + if not isinstance(node, dict): + return + fmt = node.get("format") + if isinstance(fmt, str): + formats.add(fmt) + ref = node.get("$ref") + if isinstance(ref, str): + target_id = ref.split("#", 1)[0] + if target_id and target_id not in visited_ids and target_id in by_id: + visited_ids.add(target_id) + walk(by_id[target_id]) + for key in _SUBSCHEMA_KEYS: + walk(node.get(key)) + for key in _SUBSCHEMA_LIST_KEYS: + items = node.get(key) + if isinstance(items, list): + for item in items: + walk(item) + for key in _SUBSCHEMA_MAP_KEYS: + mapping = node.get(key) + if isinstance(mapping, dict): + for sub in mapping.values(): + walk(sub) + + walk(schema) + return formats + + +def _format_checker(schema: dict) -> FormatChecker: + """A FormatChecker verified to support every `format` declared by `schema`.""" + checker = FormatChecker() + required = _required_formats(schema, _schemas_by_id()) + missing = sorted(fmt for fmt in required if fmt not in checker.checkers) + if missing: + names = ", ".join(missing) + raise FormatCheckerError( + f"missing jsonschema format checker(s) for {names}; " + "install rfc3339-validator and rfc3986-validator for the shipped schemas, " + "or jsonschema[format-nongpl] (or the format's validator package) for custom $BENCHX_SCHEMAS" + ) + return checker + + @cache def lookup(kind: str, version: str) -> Draft202012Validator | None: named = release(kind, version) @@ -44,7 +128,7 @@ def lookup(kind: str, version: str) -> Draft202012Validator | None: return None return Draft202012Validator( - schema, format_checker=FormatChecker(), registry=_registry() + schema, format_checker=_format_checker(schema), registry=_registry() ) @@ -57,7 +141,7 @@ def fragment(kind: str, release: str, pointer: str) -> Draft202012Validator | No return Draft202012Validator( {"$ref": f"{schema['$id']}#{pointer}"}, - format_checker=FormatChecker(), + format_checker=_format_checker(schema), registry=_registry(), ) @@ -81,8 +165,7 @@ def _read(kind: str, release: str) -> dict | None: @cache -def _registry() -> Registry: - # Schemas refer to each other by $id, so each validator sees all of them. +def _schemas_by_id() -> dict[str, dict]: schemas = [ _read(kind.name, release.name) for kind in sorted(SCHEMAS.iterdir()) @@ -90,8 +173,26 @@ def _registry() -> Registry: for release in sorted(kind.iterdir()) if release.is_dir() ] - return Registry().with_resources( - (schema["$id"], Resource.from_contents(schema)) + return { + schema["$id"]: schema for schema in schemas - if schema is not None + if schema is not None and "$id" in schema + } + + +@cache +def _registry() -> Registry: + # Schemas refer to each other by $id, so each validator sees all of them. + return Registry().with_resources( + (schema_id, Resource.from_contents(schema)) + for schema_id, schema in _schemas_by_id().items() ) + + +def clear_cache() -> None: + """Reset cached schemas and validators (for tests that change SCHEMAS or FormatChecker).""" + lookup.cache_clear() + fragment.cache_clear() + _read.cache_clear() + _schemas_by_id.cache_clear() + _registry.cache_clear() diff --git a/src/benchx/core/errors.py b/src/benchx/core/errors.py index 432beeba..3eaceb30 100644 --- a/src/benchx/core/errors.py +++ b/src/benchx/core/errors.py @@ -5,11 +5,18 @@ from enum import StrEnum from typing import ClassVar -from jsonpointer import JsonPointer - PathParts = tuple[str | int, ...] +def _json_pointer(parts: PathParts) -> str: + """Format path components as an RFC 6901 JSON Pointer, or '/' for the root.""" + if not parts: + return "/" + return "/" + "/".join( + str(part).replace("~", "~0").replace("/", "~1") for part in parts + ) + + class Code(StrEnum): MALFORMED = "malformed" IDENTITY_VIOLATION = "identity-violation" @@ -31,7 +38,7 @@ def message(self) -> str: @property def location(self) -> str | None: - return JsonPointer.from_parts(self.path).path or "/" + return _json_pointer(self.path) def __str__(self) -> str: if self.location is None: diff --git a/src/benchx/parquet.py b/src/benchx/parquet.py index 352247dd..883dc77d 100644 --- a/src/benchx/parquet.py +++ b/src/benchx/parquet.py @@ -9,7 +9,15 @@ from datetime import datetime -import pyarrow as pa +try: + import pyarrow as pa +except ModuleNotFoundError as exc: + if exc.name and exc.name.split(".")[0] == "pyarrow": + raise ModuleNotFoundError( + "Parquet support requires pyarrow; install it with `pip install 'benchx[parquet]'`.", + name="pyarrow", + ) from exc + raise import rfc8785 from .core import catalog diff --git a/src/benchx/results.py b/src/benchx/results.py new file mode 100644 index 00000000..332a734d --- /dev/null +++ b/src/benchx/results.py @@ -0,0 +1,202 @@ +"""Arrow-free result helpers and in-memory ingestion (schema §4.1, §5.3, §5.4). + +Shared between the Parquet-backed `Store` (`benchx.store`) and the JSON-only +paths (`bx bench --no-ingest`, `bx compare --results`) so that running and +comparing JSON results never imports PyArrow while enforcing the same +validation, deduplication, and consistency checks. +""" + +import json +from pathlib import Path + +from . import core, identity +from .core import loader +from .core.errors import ( + AttemptConflict, + DocumentError, + IdempotencyConflict, + ReadError, + UnitConflict, +) + +# Schema §5.3 attempt integrity: what results sharing an attempt_key agree on. +_ATTEMPT_FIELDS = ("project", "source", "revision", "benchmark") +_ATTEMPT_COORDINATES = ("workload", "subject", "environment") +_ATTEMPT_PROVENANCE = ( + "run_key", + "subject_dirty", + "subject_tree", + "benchmark_dirty", + "benchmark_tree", +) + + +def attempt_signature(doc: dict) -> str: + facts = {k: doc.get(k) for k in _ATTEMPT_FIELDS} + facts.update({k: doc["coordinates"][k] for k in _ATTEMPT_COORDINATES}) + facts["environment"] = { + k: facts["environment"][k] for k in ("schema_version", "identity") + } + facts.update({k: doc["provenance"].get(k) for k in _ATTEMPT_PROVENANCE}) + return core.sha256_hex(core.canonical(facts)) + + +def store_project(doc: dict) -> str: + """The project, or local/ for a thin result (schema §4.1).""" + if "project" in doc: + return doc["project"] + ident = doc["coordinates"]["environment"]["identity"] + return f"local/{ident.get('runner') or ident.get('hostname') or 'unknown'}" + + +def warning_codes(document: dict) -> list[str]: + """A result's quality warnings by code: a warning is a token, or an object that has a `code`.""" + return [ + w.get("code", str(w)) if isinstance(w, dict) else str(w) + for w in document.get("quality", {}).get("warnings", []) + ] + + +def rejection(name: str, error: DocumentError) -> dict: + where = f"at {error.location}: " if error.location else "" + return {"document": name, "code": str(error.code), "message": where + error.message} + + +def sweep_paths(paths) -> tuple[list[tuple[str, str]], dict[str, dict], list[dict]]: + """Sweep files and directories of JSON documents in transit: work orders + are separated from results, and unreadable files become rejections (§5.4).""" + files = [] + for path in map(Path, paths): + files.extend(sorted(path.glob("*.json")) if path.is_dir() else [path]) + orders, documents, unreadable = {}, [], [] + for f in files: + # A file that cannot be read is one rejection, not the end of the sweep (§5.4). + text = loader.read_text(f) + if isinstance(text, ReadError): + unreadable.append(rejection(f.name, text)) + continue + if not f.name.startswith("workorder-"): + documents.append((f.name, text)) + continue + order, errors = loader.parse(text) + if order is None: + unreadable.append(rejection(f.name, errors[0])) + else: + orders[core.order_ref(order)] = order + return documents, orders, unreadable + + +def ingest_documents( + documents: list[tuple[str, str]], + orders: dict | None = None, + *, + existing: dict[tuple[str, str], str] | None = None, + attempts: dict[str, str] | None = None, + units: dict[tuple[str, str], str] | None = None, +) -> tuple[list[dict], list[dict], dict]: + """Validate and check per-result outcomes; a batch is N independent documents (§5.4). + + Returns `(accepted, derived, outcome)` and updates `existing`, `attempts`, and + `units` in place when provided. + """ + orders = orders or {} + existing = {} if existing is None else existing + attempts = {} if attempts is None else attempts + units = {} if units is None else units + + accepted, derived = [], [] + outcome = {"ingested": 0, "duplicate": 0, "rejected": []} + for name, text in documents: + doc, errors = loader.parse(text) + if doc is not None: + errors = core.check(doc, "measurement-result") + if errors: + outcome["rejected"].append(rejection(name, errors[0])) + continue + + project = store_project(doc) + canonical = core.canonical(doc) + payload = core.sha256_hex(canonical) + key = (doc["producer"]["name"], doc["ingest_key"]) + quantity = doc["measurement"]["quantity"] + known = units.get((project, quantity["name"])) + signature = attempt_signature(doc) + if key in existing and existing[key] == payload: + outcome["duplicate"] += 1 + continue + if key in existing: + conflict = IdempotencyConflict() + elif known and known != quantity["unit"]: + conflict = UnitConflict( + quantity=quantity["name"], + project=project, + known=known, + reported=quantity["unit"], + ) + elif attempts.get(doc["attempt_key"], signature) != signature: + conflict = AttemptConflict(attempt_key=doc["attempt_key"]) + else: + conflict = None + if conflict is not None: + outcome["rejected"].append(rejection(name, conflict)) + continue + + existing[key] = payload + attempts[doc["attempt_key"]] = signature + units[(project, quantity["name"])] = quantity["unit"] # first sight defines the unit + ref = doc["provenance"].get("info", {}).get("workorder_ref") + order = orders.get(ref) + accepted.append(doc) + derived.append({ + "store_project": project, + "reported_coordinates_fingerprint": identity.reported_coordinates_fingerprint(doc), + "payload_sha256": payload, + "attempt_signature": signature, + "document": canonical.decode(), + "work_order_ref": ref, + "work_order": core.canonical(order).decode() if order is not None else None, + }) + outcome["ingested"] += 1 + + return accepted, derived, outcome + + +class MemoryStore: + """In-memory result store enforcing the same ingest contract as `Store`, + without requiring PyArrow or writing Parquet files.""" + + def __init__(self) -> None: + self._existing: dict[tuple[str, str], str] = {} + self._attempts: dict[str, str] = {} + self._units: dict[tuple[str, str], str] = {} + self._documents: list[dict] = [] + self._orders: dict[str, dict] = {} + + def documents(self, run_key: str) -> list[dict]: + """The accepted documents of one run, in canonical form.""" + return [d for d in self._documents if d["provenance"]["run_key"] == run_key] + + def order(self, ref: str) -> dict | None: + return self._orders.get(ref) + + def ingest_paths(self, paths) -> dict: + """Sweep files and directories of JSON documents in transit.""" + documents, orders, unreadable = sweep_paths(paths) + outcome = self.ingest(documents, orders) + outcome["rejected"] = unreadable + outcome["rejected"] + return outcome + + def ingest(self, documents: list[tuple[str, str]], orders: dict | None = None) -> dict: + """Per-result outcomes; a batch is N independent documents (§5.4).""" + _, derived, outcome = ingest_documents( + documents, + orders, + existing=self._existing, + attempts=self._attempts, + units=self._units, + ) + for row in derived: + self._documents.append(json.loads(row["document"])) + if row["work_order_ref"] is not None and row["work_order"] is not None: + self._orders.setdefault(row["work_order_ref"], json.loads(row["work_order"])) + return outcome diff --git a/src/benchx/store.py b/src/benchx/store.py index bb13b81f..90ee4814 100644 --- a/src/benchx/store.py +++ b/src/benchx/store.py @@ -18,34 +18,12 @@ import statistics from pathlib import Path -import pyarrow as pa -import pyarrow.parquet as pq - from . import core, identity, parquet -from .core import loader -from .core.errors import AttemptConflict, DocumentError, IdempotencyConflict, ReadError, UnitConflict from .projection import Projection +from .results import ingest_documents, store_project, sweep_paths, warning_codes -# Schema §5.3 attempt integrity: what results sharing an attempt_key agree on. -_ATTEMPT_FIELDS = ("project", "source", "revision", "benchmark") -_ATTEMPT_COORDINATES = ("workload", "subject", "environment") -_ATTEMPT_PROVENANCE = ("run_key", "subject_dirty", "subject_tree", "benchmark_dirty", "benchmark_tree") - - -def _attempt_signature(doc: dict) -> str: - facts = {k: doc.get(k) for k in _ATTEMPT_FIELDS} - facts.update({k: doc["coordinates"][k] for k in _ATTEMPT_COORDINATES}) - facts["environment"] = {k: facts["environment"][k] for k in ("schema_version", "identity")} - facts.update({k: doc["provenance"].get(k) for k in _ATTEMPT_PROVENANCE}) - return core.sha256_hex(core.canonical(facts)) - - -def store_project(doc: dict) -> str: - """The project, or local/ for a thin result (schema §4.1).""" - if "project" in doc: - return doc["project"] - ident = doc["coordinates"]["environment"]["identity"] - return f"local/{ident.get('runner') or ident.get('hostname') or 'unknown'}" +import pyarrow as pa +import pyarrow.parquet as pq class Store: @@ -170,85 +148,20 @@ def history(self, fingerprint_prefix: str, projection: Projection = identity.POL def ingest_paths(self, paths) -> dict: """Sweep files and directories of JSON documents in transit: work orders are attached to the results that reference them, results ingested.""" - files = [] - for path in map(Path, paths): - files.extend(sorted(path.glob("*.json")) if path.is_dir() else [path]) - orders, documents, unreadable = {}, [], [] - for f in files: - # A file that cannot be read is one rejection, not the end of the sweep (§5.4). - text = loader.read_text(f) - if isinstance(text, ReadError): - unreadable.append(_rejection(f.name, text)) - continue - if not f.name.startswith("workorder-"): - documents.append((f.name, text)) - continue - order, errors = loader.parse(text) - if order is None: - unreadable.append(_rejection(f.name, errors[0])) - else: - orders[core.order_ref(order)] = order + documents, orders, unreadable = sweep_paths(paths) outcome = self.ingest(documents, orders) outcome["rejected"] = unreadable + outcome["rejected"] return outcome def ingest(self, documents: list[tuple[str, str]], orders: dict | None = None) -> dict: """Per-result outcomes; a batch is N independent documents (§5.4).""" - orders = orders or {} index = self._index() existing = {(r["producer"], r["ingest_key"]): r["payload_sha256"] for r in index} attempts = {r["attempt_key"]: r["attempt_signature"] for r in index} units = {(r["store_project"], r["quantity"]): r["unit"] for r in index} - - accepted, derived = [], [] - outcome = {"ingested": 0, "duplicate": 0, "rejected": []} - for name, text in documents: - doc, errors = loader.parse(text) - if doc is not None: - errors = core.check(doc, "measurement-result") - if errors: - outcome["rejected"].append(_rejection(name, errors[0])) - continue - - project = store_project(doc) - canonical = core.canonical(doc) - payload = core.sha256_hex(canonical) - key = (doc["producer"]["name"], doc["ingest_key"]) - quantity = doc["measurement"]["quantity"] - known = units.get((project, quantity["name"])) - signature = _attempt_signature(doc) - if key in existing and existing[key] == payload: - outcome["duplicate"] += 1 - continue - if key in existing: - conflict = IdempotencyConflict() - elif known and known != quantity["unit"]: - conflict = UnitConflict(quantity=quantity["name"], project=project, - known=known, reported=quantity["unit"]) - elif attempts.setdefault(doc["attempt_key"], signature) != signature: - conflict = AttemptConflict(attempt_key=doc["attempt_key"]) - else: - conflict = None - if conflict is not None: - outcome["rejected"].append(_rejection(name, conflict)) - continue - - existing[key] = payload - units[(project, quantity["name"])] = quantity["unit"] # first sight defines the unit - ref = doc["provenance"].get("info", {}).get("workorder_ref") - order = orders.get(ref) - accepted.append(doc) - derived.append({ - "store_project": project, - "reported_coordinates_fingerprint": identity.reported_coordinates_fingerprint(doc), - "payload_sha256": payload, - "attempt_signature": signature, - "document": canonical.decode(), - "work_order_ref": ref, - "work_order": core.canonical(order).decode() if order is not None else None, - }) - outcome["ingested"] += 1 - + accepted, derived, outcome = ingest_documents( + documents, orders, existing=existing, attempts=attempts, units=units + ) if accepted: self._write(parquet.results_table(accepted, derived)) return outcome @@ -267,11 +180,6 @@ def _write(self, new: pa.Table) -> None: os.replace(tmp, self.path) -def _rejection(name: str, error: DocumentError) -> dict: - where = f"at {error.location}: " if error.location else "" - return {"document": name, "code": str(error.code), "message": where + error.message} - - _ENTITY_KEYS = { "project": lambda r: r["key"], "source": lambda r: (r["project"], r["uri"]), @@ -284,12 +192,6 @@ def _rejection(name: str, error: DocumentError) -> dict: } -def warning_codes(document: dict) -> list[str]: - """A result's quality warnings by code: a warning is a token, or an object that has a `code`.""" - return [w.get("code", str(w)) if isinstance(w, dict) else str(w) - for w in document.get("quality", {}).get("warnings", [])] - - def _add_entities(entities: dict, doc: dict, project: str, projection: Projection) -> None: def put(name, row): entities.setdefault(name, {}).setdefault(_ENTITY_KEYS[name](row), row) diff --git a/src/benchx/tables.py b/src/benchx/tables.py index 96e01782..376a6471 100644 --- a/src/benchx/tables.py +++ b/src/benchx/tables.py @@ -12,8 +12,8 @@ import statistics from .compare import WORKLOAD, _c, _estimate, _run_documents, _unit_order, _workload, profile_projection +from .results import store_project from .snapshot import DIRTY_WARNINGS -from .store import store_project ROLES = ("baseline", "contender") CHANGED = {"regressed": "+", "improved": "-", "changed": "~"} # as asv continuous marks them diff --git a/src/benchx/views/__init__.py b/src/benchx/views/__init__.py new file mode 100644 index 00000000..a905b690 --- /dev/null +++ b/src/benchx/views/__init__.py @@ -0,0 +1,45 @@ +"""Static dashboard views and local report server (`bx view`). + +Derives flat tables (marts) and view JSON payloads from a local Parquet store, +JSON result files, or a CI `runs///` directory, and bundles a static +HTML/JS dashboard that can be served locally or deployed to static hosting. +""" + +from .inputs import ViewError, parse_time, read_results, read_runs_dir, read_runs_metadata, read_store, run_schema, write_run +from .marts import MOVERS_PER_COMMIT, RECENT, build_changes, build_commits, build_latest, build_marts, build_movers, build_points, write_parquet_marts +from .payloads import MOVERS, NOISIEST, NOTABLE, RECENT_COMMITS, SPARK_POINTS, build_views, last_executed, newest_commits, signed +from .site import STATIC_DIR, build_site, create_server, resolve_project, serve_site + +__all__ = [ + "MOVERS", + "MOVERS_PER_COMMIT", + "NOISIEST", + "NOTABLE", + "RECENT", + "RECENT_COMMITS", + "SPARK_POINTS", + "STATIC_DIR", + "ViewError", + "build_changes", + "build_commits", + "build_latest", + "build_marts", + "build_movers", + "build_points", + "build_site", + "build_views", + "create_server", + "last_executed", + "newest_commits", + "parse_time", + "read_results", + "read_runs_dir", + "read_runs_metadata", + "read_store", + "resolve_project", + "run_schema", + "serve_site", + "signed", + "write_parquet_marts", + "write_run", +] diff --git a/src/benchx/views/inputs.py b/src/benchx/views/inputs.py new file mode 100644 index 00000000..848d2a7b --- /dev/null +++ b/src/benchx/views/inputs.py @@ -0,0 +1,594 @@ +"""Read benchmark results for the views layer from a local store, JSON files, or a CI runs/ tree. + +Three inputs, one normalized shape (`runs`, `raw_points`, `ingest_problems`): + + runs/ directory runs///{run.parquet, results.parquet} (CI / node layout) + local store store.parquet (benchx.store.Store) + JSON results result files and work orders, read via MemoryStore without PyArrow + +PyArrow is imported lazily only when reading or writing Parquet files, so JSON-only +views work with a minimal `pip install benchx`. +""" + +from __future__ import annotations + +import datetime as dt +import json +import re +import subprocess +from collections import defaultdict +from pathlib import Path + +import rfc8785 + +from .. import identity, projection +from ..results import MemoryStore + +UTC = dt.timezone.utc +_HEX_RE = re.compile(r"^[0-9a-fA-F]{7,40}$") +_URI_SCHEME_RE = re.compile(r"^[a-zA-Z][a-zA-Z0-9+.-]*://") +_PATH_UNSAFE_RE = re.compile(r'[/\\\x00:*?"<>|]+') +_DOTS_RUN_RE = re.compile(r"\.\.+") + + +class ViewError(Exception): + """The view inputs could not be read or built.""" + + +def check_local_path(value: str | Path) -> Path: + """Reject remote URIs (such as s3:// or https://) before turning them into Paths.""" + text = str(value) + if _URI_SCHEME_RE.match(text): + raise ViewError( + f"remote URI {text!r} is not supported directly; " + "sync the files locally first (for example with `aws s3 sync`)" + ) + return Path(value) + + +def safe_path_segment(value: str) -> str: + """Flatten path separators and traversal dots so generated filenames stay inside out/.""" + cleaned = _DOTS_RUN_RE.sub("_", _PATH_UNSAFE_RE.sub("_", str(value))) + return "_" if cleaned in ("", ".") else cleaned + + +def short_sha(sha: str) -> str: + """Short display label for a commit SHA, preserving a visible marker for dirty trees.""" + if "-dirty-" in sha: + base, tree = sha.split("-dirty-", 1) + return f"{base[:7]}+dirty-{tree[:4]}" + return sha[:12] + + +def _require_pyarrow(feature: str = "Parquet view support"): + try: + import pyarrow as pa + import pyarrow.parquet as pq + except ModuleNotFoundError as exc: + if exc.name and exc.name.split(".")[0] == "pyarrow": + raise ModuleNotFoundError( + f"{feature} requires pyarrow; install it with `pip install 'benchx[parquet]'`.", + name="pyarrow", + ) from exc + raise + return pa, pq + + +def parse_time(value) -> dt.datetime | None: + """Parse an ISO-8601 timestamp or datetime into a UTC datetime (or None).""" + if value is None or value == "": + return None + if isinstance(value, dt.datetime): + return value.astimezone(UTC) if value.tzinfo else value.replace(tzinfo=UTC) + text = str(value).strip() + if not text: + return None + if "T" not in text and " " in text: + text = text.replace(" ", "T", 1) + if text.endswith("Z"): + text = text[:-1] + "+00:00" + parsed = dt.datetime.fromisoformat(text) + return parsed.astimezone(UTC) if parsed.tzinfo else parsed.replace(tzinfo=UTC) + + +def run_schema(): + """Arrow schema for runs///run.parquet.""" + pa, _ = _require_pyarrow("run.parquet") + ts = pa.timestamp("us", tz="UTC") + return pa.schema([ + ("sha", pa.string()), + ("parents", pa.list_(pa.string())), + ("author_date", ts), + ("committer_date", ts), + ("subject", pa.string()), + ("branch", pa.string()), + ("node", pa.string()), + ("state", pa.string()), + ("reason", pa.string()), + ("started_at", ts), + ("finished_at", ts), + ("suites_ok", pa.int32()), + ("suites_failed", pa.int32()), + ("results", pa.int32()), + ("ingest_problems", pa.int32()), + ("log", pa.string()), + ]) + + +def write_run( + node_dir: Path, + commit: dict, + status: dict, + log: str = "", + ingest_problems: int = 0, +) -> Path: + """Write one run's `run.parquet` from commit metadata, status, and log text.""" + pa, pq = _require_pyarrow("write_run") + suites = status.get("suites") or {} + node_dir = Path(node_dir) + node_dir.mkdir(parents=True, exist_ok=True) + row = { + "sha": commit["sha"], + "parents": list(commit.get("parents") or []), + "author_date": parse_time(commit.get("author_date")), + "committer_date": parse_time(commit.get("committer_date")), + "subject": commit.get("subject") or "", + "branch": commit.get("branch") or "main", + "node": status.get("node") or node_dir.name, + "state": status.get("state") or "unknown", + "reason": status.get("reason"), + "started_at": parse_time(status.get("started_at")), + "finished_at": parse_time(status.get("finished_at")), + "suites_ok": int(status.get("suites_ok") or 0), + "suites_failed": int(status.get("suites_failed") or 0), + "results": int(sum((s or {}).get("results", 0) for s in suites.values())), + "ingest_problems": int(ingest_problems), + "log": log, + } + out = node_dir / "run.parquet" + pq.write_table(pa.Table.from_pylist([row], schema=run_schema()), out, compression="zstd") + return out + + +def read_runs_metadata(runs_dir: Path) -> list[dict]: + """Read every `//run.parquet` row under `runs_dir`.""" + _, pq = _require_pyarrow("reading run.parquet") + rows = [] + for path in sorted(Path(runs_dir).glob("*/*/run.parquet")): + table = pq.ParquetFile(path).read() + if table.num_rows: + row = table.to_pylist()[0] + for key in ("author_date", "committer_date", "started_at", "finished_at"): + row[key] = parse_time(row.get(key)) + row["parents"] = list(row.get("parents") or []) + row["log"] = row.get("log") or "" + row["ingest_problems"] = int(row.get("ingest_problems") or 0) + rows.append(row) + return rows + + +def _parse_json_obj(value) -> dict: + if value is None: + return {} + if isinstance(value, dict): + return value + if isinstance(value, str): + try: + parsed = json.loads(value) + except json.JSONDecodeError: + return {} + return parsed if isinstance(parsed, dict) else {} + return {} + + +def _canonical_params(value) -> str: + if value is None or value == "" or value == "{}": + return "" + if isinstance(value, str): + return "" if value == "{}" else value + if isinstance(value, dict): + return "" if not value else rfc8785.dumps(value).decode() + return str(value) + + +def _extract_suite(work_order: dict, doc_or_row: dict) -> str: + suites = work_order.get("suites") if isinstance(work_order, dict) else None + if isinstance(suites, list) and suites: + first = suites[0] + if isinstance(first, dict) and first.get("suite"): + return str(first["suite"]).rsplit("/", 1)[-1] + provenance = doc_or_row.get("provenance") or {} + info = _parse_json_obj(provenance.get("info")) + if info.get("suite"): + return str(info["suite"]).rsplit("/", 1)[-1] + return "default" + + +def _extract_node(run_key: str, env_identity: dict) -> str: + # A CI/node run key has the form //; keep that node name + # because it is what runs/ is filed under even when a runner renames its hostname. + if run_key and run_key.count("/") >= 2: + return run_key.split("/")[-1] + runner = env_identity.get("runner") or env_identity.get("hostname") + return str(runner) if runner else "local" + + +def _extract_estimate(measurement: dict) -> tuple[list[float], float | None, float | None, float | None, float | None]: + """Extract observation values and (estimate, mean, min, max) following identity.points().""" + obs = measurement.get("observations") or [] + vals: list[float] = [] + for item in obs: + if isinstance(item, dict): + if item.get("included") is False: + continue + v = item.get("value") + else: + v = item + if v is not None: + vals.append(float(v)) + if vals: + est = projection.estimate(identity.MEDIAN, vals) + return vals, float(est) if est is not None else None, sum(vals) / len(vals), min(vals), max(vals) + + # Producer-supplied summaries (schema §4.5): {"type": "estimate", "estimator": {"name": ...}, "value": ...}. + summaries = measurement.get("summaries") or [] + estimates: dict[str, float] = {} + first_estimate: float | None = None + stats: dict[str, float] = {} + if isinstance(summaries, list): + for s in summaries: + if not isinstance(s, dict) or s.get("value") is None: + continue + val = float(s["value"]) + if s.get("type") == "estimate": + if first_estimate is None: + first_estimate = val + name = (s.get("estimator") or {}).get("name") + if name and name not in estimates: + estimates[str(name)] = val + elif s.get("type") == "statistic" and s.get("name"): + stats.setdefault(str(s["name"]), val) + + est = estimates.get("median", estimates.get("mean", first_estimate)) + if est is None: + return [], None, None, None, None + mean = estimates.get("mean", stats.get("mean", est)) + lo = stats.get("min", est) + hi = stats.get("max", est) + return [], est, mean, lo, hi + + +def _point_from_row(row: dict, *, distinguish_dirty: bool) -> dict | None: + measurement = row.get("measurement") or {} + kind = measurement.get("kind") + if kind is not None and kind != "measurand": + return None + + revision = row.get("revision") or {} + base_sha = str(revision.get("key") or "") + provenance = row.get("provenance") or {} + subject_dirty = provenance.get("subject_dirty") + subject_tree = provenance.get("subject_tree") + if distinguish_dirty and subject_dirty == "dirty" and subject_tree: + sha = f"{base_sha[:12]}-dirty-{str(subject_tree)[:8]}" + dirty_parent = base_sha + else: + sha = base_sha + dirty_parent = None + + run_key = str(provenance.get("run_key") or "") + coordinates = row.get("coordinates") or {} + workload = coordinates.get("workload") or {} + env = coordinates.get("environment") or {} + env_identity = _parse_json_obj(env.get("identity")) + work_order = _parse_json_obj(row.get("work_order")) + + quantity = measurement.get("quantity") or {} + vals, estimate, mean, lo, hi = _extract_estimate(measurement) + + return { + "sha": sha, + "node": _extract_node(run_key, env_identity), + "suite": _extract_suite(work_order, row), + "case": str(workload.get("name") or ""), + "params": _canonical_params(workload.get("parameters")), + "quantity": str(quantity.get("name") or ""), + "unit": str(quantity.get("unit") or ""), + "direction": str(quantity.get("direction") or ""), + "status": str(measurement.get("status") or "unknown"), + "reason": str(measurement.get("reason") or ""), + "estimate": estimate, + "mean": mean, + "min": lo, + "max": hi, + "n": len(vals), + "started_at": parse_time(provenance.get("started_at")), + "run_key": run_key, + "ingest_key": str(row.get("ingest_key") or ""), + "_vals": vals, + "_dirty_parent": dirty_parent, + } + + +_PARQUET_POINT_COLUMNS = [ + "revision", + "coordinates", + "measurement", + "provenance", + "ingest_key", + "work_order", +] + + +def _read_parquet_points(path: Path, *, distinguish_dirty: bool) -> list[dict]: + _, pq = _require_pyarrow("reading Parquet results") + pf = pq.ParquetFile(path) + available = set(pf.schema_arrow.names) + cols = [c for c in _PARQUET_POINT_COLUMNS if c in available] + if not cols: + return [] + table = pf.read(columns=cols) + points = [] + for row in table.to_pylist(): + pt = _point_from_row(row, distinguish_dirty=distinguish_dirty) + if pt is not None: + points.append(pt) + return points + + +def _collapse_duplicate_points(points: list[dict]) -> list[dict]: + """Combine multiple results for the same (sha, node, suite, case, params, quantity). + + In CI runs/ each commit has one result per series so this is a no-op. In a local + store populated by `bx compare --rounds R`, combining rounds yields one point per + commit with pooled observations instead of duplicate commit_order entries. + """ + grouped: dict[tuple, list[dict]] = defaultdict(list) + for pt in points: + key = (pt["sha"], pt["node"], pt["suite"], pt["case"], pt["params"], pt["quantity"]) + grouped[key].append(pt) + + out: list[dict] = [] + for group in grouped.values(): + if len(group) == 1: + pt = dict(group[0]) + pt.pop("_vals", None) + pt.pop("_dirty_parent", None) + out.append(pt) + continue + + group.sort(key=lambda p: (p["started_at"] or dt.datetime.min.replace(tzinfo=UTC), p["ingest_key"])) + measured = [p for p in group if p["status"] in ("success", "partial") and p["estimate"] is not None] + if measured: + vals: list[float] = [] + for p in measured: + vals.extend(p.get("_vals") or ([p["estimate"]] if p["estimate"] is not None else [])) + latest = measured[-1] + merged = dict(latest) + est = projection.estimate(identity.MEDIAN, vals) + merged["estimate"] = float(est) if est is not None else None + merged["mean"] = sum(vals) / len(vals) + merged["min"] = min(vals) + merged["max"] = max(vals) + merged["n"] = sum(p["n"] for p in measured) + else: + merged = dict(group[-1]) + merged.pop("_vals", None) + merged.pop("_dirty_parent", None) + out.append(merged) + return out + + +def _git_commit_info(shas: list[str], repo: Path) -> dict[str, dict]: + """Batch-read commit parents, timestamps, and subject from a local git repo.""" + candidates = sorted({s for s in shas if _HEX_RE.match(s)}) + if not candidates or not repo.exists(): + return {} + try: + proc = subprocess.run( + ["git", "-C", str(repo), "cat-file", "--batch"], + input="".join(f"{sha}\n" for sha in candidates).encode("utf-8"), + capture_output=True, + check=False, + ) + except OSError: + return {} + if proc.returncode != 0: + return {} + + data = proc.stdout + pos = 0 + size = len(data) + info: dict[str, dict] = {} + while pos < size: + nl = data.find(b"\n", pos) + if nl == -1: + break + header = data[pos:nl].decode("utf-8", errors="replace") + pos = nl + 1 + parts = header.split() + if len(parts) >= 2 and parts[1] == "missing": + continue + if len(parts) < 3: + break + obj_sha, obj_type, obj_len = parts[0], parts[1], int(parts[2]) + body = data[pos:pos + obj_len] + pos += obj_len + 1 # trailing newline after object body + if obj_type != "commit": + continue + info[obj_sha] = _parse_git_commit(body.decode("utf-8", errors="replace")) + by_requested: dict[str, dict] = {} + for sha in candidates: + if sha in info: + by_requested[sha] = info[sha] + else: + for full_sha, meta in info.items(): + if full_sha.startswith(sha): + by_requested[sha] = meta + break + return by_requested + + +def _parse_git_commit(text: str) -> dict: + headers_text, _, message = text.partition("\n\n") + parents: list[str] = [] + author_date: dt.datetime | None = None + committer_date: dt.datetime | None = None + for line in headers_text.splitlines(): + if line.startswith("parent "): + parents.append(line.split(" ", 1)[1].strip()) + elif line.startswith("author "): + author_date = _parse_git_ident_time(line) + elif line.startswith("committer "): + committer_date = _parse_git_ident_time(line) + subject = message.splitlines()[0].strip() if message.strip() else "" + return { + "parents": parents, + "author_date": author_date, + "committer_date": committer_date, + "subject": subject, + } + + +def _parse_git_ident_time(line: str) -> dt.datetime | None: + parts = line.rsplit(" ", 2) + if len(parts) < 3: + return None + try: + epoch = int(parts[1]) + except ValueError: + return None + return dt.datetime.fromtimestamp(epoch, tz=UTC) + + +def _find_git_repo(start: Path | None) -> Path | None: + if start is None: + return None + cur = Path(start).resolve() + if cur.is_file(): + cur = cur.parent + for directory in (cur, *cur.parents): + if (directory / ".git").exists(): + return directory + return None + + +def synthesize_runs( + raw_points: list[dict], + *, + repo: Path | None = None, + default_state: str = "finished", +) -> list[dict]: + """Synthesize per-(sha, node) run rows when no `run.parquet` files exist.""" + by_pair: dict[tuple[str, str], list[dict]] = defaultdict(list) + dirty_parents: dict[str, str] = {} + for pt in raw_points: + by_pair[(pt["sha"], pt["node"])].append(pt) + if pt.get("_dirty_parent"): + dirty_parents[pt["sha"]] = pt["_dirty_parent"] + + git_repo = _find_git_repo(repo) + git_meta = _git_commit_info([sha for sha, _ in by_pair], git_repo) if git_repo else {} + + runs: list[dict] = [] + for (sha, node), pts in sorted(by_pair.items()): + times = [p["started_at"] for p in pts if p["started_at"] is not None] + started_at = min(times) if times else None + finished_at = max(times) if times else None + + suite_ok: dict[str, bool] = {} + for p in pts: + suite_ok[p["suite"]] = suite_ok.get(p["suite"], False) or (p["status"] in ("success", "partial")) + suites_ok = sum(1 for ok in suite_ok.values() if ok) + suites_failed = sum(1 for ok in suite_ok.values() if not ok) + + meta = git_meta.get(sha, {}) + if sha in dirty_parents: + base = dirty_parents[sha] + parents = [base] + subject = f"dirty working tree on {base[:12]}" + committer_date = started_at + author_date = started_at + else: + parents = meta.get("parents") or [] + subject = meta.get("subject") or "" + committer_date = meta.get("committer_date") or started_at + author_date = meta.get("author_date") or committer_date + + runs.append({ + "sha": sha, + "parents": parents, + "author_date": author_date, + "committer_date": committer_date, + "subject": subject, + "branch": "", + "node": node, + "state": default_state, + "reason": None, + "started_at": started_at, + "finished_at": finished_at, + "suites_ok": suites_ok, + "suites_failed": suites_failed, + "results": len(pts), + "ingest_problems": 0, + "log": "", + }) + return runs + + +def read_runs_dir(runs_dir: str | Path) -> tuple[list[dict], list[dict], int]: + """Read a CI/node `runs///{run.parquet, results.parquet}` tree.""" + runs_path = check_local_path(runs_dir) + _require_pyarrow("reading runs/ Parquet files") + if not runs_path.is_dir(): + raise ViewError(f"runs directory not found: {runs_path}") + raw_runs = read_runs_metadata(runs_path) + dedup_runs: dict[tuple[str, str], dict] = {} + for run in raw_runs: + dedup_runs[(run["sha"], run["node"])] = run + runs = list(dedup_runs.values()) + ingest_problems = sum(r["ingest_problems"] for r in runs) + raw_points: list[dict] = [] + for path in sorted(runs_path.glob("*/*/results.parquet")): + raw_points.extend(_read_parquet_points(path, distinguish_dirty=False)) + return runs, _collapse_duplicate_points(raw_points), ingest_problems + + +def read_store(store_path: str | Path, *, repo: Path | None = None) -> tuple[list[dict], list[dict], int]: + """Read a single local benchx Parquet store (`store.parquet`).""" + path = check_local_path(store_path) + _require_pyarrow("reading a Parquet store") + if not path.is_file(): + raise ViewError(f"store not found: {path}") + raw_points = _read_parquet_points(path, distinguish_dirty=True) + collapsed = _collapse_duplicate_points(raw_points) + runs = synthesize_runs(raw_points, repo=repo) + counts_by_pair: dict[tuple[str, str], int] = defaultdict(int) + for pt in collapsed: + counts_by_pair[(pt["sha"], pt["node"])] += 1 + for run in runs: + run["results"] = counts_by_pair[(run["sha"], run["node"])] + return runs, collapsed, 0 + + +def read_results(paths: list[str | Path], *, repo: Path | None = None) -> tuple[list[dict], list[dict], int, dict]: + """Read JSON result files and work orders via MemoryStore without PyArrow.""" + local_paths = [check_local_path(p) for p in paths] + store = MemoryStore() + outcome = store.ingest_paths(local_paths) + raw_points: list[dict] = [] + for doc in store._documents: + ref = (doc.get("provenance") or {}).get("info", {}).get("workorder_ref") + order = store.order(ref) if ref else None + row = dict(doc, work_order=order) + pt = _point_from_row(row, distinguish_dirty=True) + if pt is not None: + raw_points.append(pt) + collapsed = _collapse_duplicate_points(raw_points) + runs = synthesize_runs(raw_points, repo=repo) + counts_by_pair: dict[tuple[str, str], int] = defaultdict(int) + for pt in collapsed: + counts_by_pair[(pt["sha"], pt["node"])] += 1 + for run in runs: + run["results"] = counts_by_pair[(run["sha"], run["node"])] + return runs, collapsed, len(outcome["rejected"]), outcome diff --git a/src/benchx/views/marts.py b/src/benchx/views/marts.py new file mode 100644 index 00000000..8e7c1a28 --- /dev/null +++ b/src/benchx/views/marts.py @@ -0,0 +1,413 @@ +"""Derive flat, query-shaped tables (marts) from runs and benchmark points. + +Pure Python stdlib computation; PyArrow is imported lazily only when writing the +optional `.parquet` marts: + + commits.parquet one row per measured commit x node, with ancestry and status + points.parquet one row per result, sorted by suite, case and commit order + changes.parquet every point against the previous point of its series + latest.parquet one row per series: latest estimate, changes, noise, rank + movers.parquet the largest changes per commit x node + manifest.json counts, the latest commit, the tables + +A series is (suite, case, params, quantity, node). Both `success` and `partial` +results contribute estimates (matching `benchx.identity.points`), while `skipped` +is preserved as non-failing status. Verdicts are not computed here: the relative +changes are a heuristic until benchx's comparator history mode exists. +""" + +from __future__ import annotations + +import datetime as dt +import heapq +import statistics +from collections import Counter, defaultdict +from pathlib import Path + +from .inputs import UTC, _require_pyarrow, short_sha, synthesize_runs + +RECENT = 21 # points per series the noise estimate looks at (20 consecutive changes) +MOVERS_PER_COMMIT = 100 +MEASURED_STATUSES = ("success", "partial") +NON_FAILED_STATUSES = ("success", "partial", "skipped") + + +def is_measured(point: dict) -> bool: + """Whether a point carries a usable measurement estimate (matching identity.points).""" + return point["status"] in MEASURED_STATUSES and point["estimate"] is not None + + +def is_failed_status(status: str | None) -> bool: + """Whether a result status represents a failure (error/censored, not skipped/partial/success).""" + return bool(status) and status not in NON_FAILED_STATUSES + + +def _series_key(row: dict) -> tuple: + return (row["suite"], row["case"], row["params"], row["quantity"], row["node"]) + + +def _order_shas(runs: list[dict]) -> dict[str, int]: + """Order unique SHAs by committer_date while keeping every known parent before its child.""" + min_dt = dt.datetime.min.replace(tzinfo=UTC) + date_by_sha: dict[str, dt.datetime] = {} + parents_by_sha: dict[str, set[str]] = defaultdict(set) + for run in runs: + sha = run["sha"] + cdate = run.get("committer_date") or min_dt + if sha not in date_by_sha or cdate < date_by_sha[sha]: + date_by_sha[sha] = cdate + for p in run.get("parents") or (): + if p and p != sha: + parents_by_sha[sha].add(p) + + all_shas = set(date_by_sha) + in_degree: dict[str, int] = {} + children: dict[str, list[str]] = defaultdict(list) + for sha in all_shas: + known_parents = [p for p in parents_by_sha.get(sha, ()) if p in all_shas] + in_degree[sha] = len(known_parents) + for p in known_parents: + children[p].append(sha) + + # Kahn's topological sort with a min-heap keyed on (committer_date, sha): + # parents always precede their children even when timestamps tie within one second. + ready: list[tuple[dt.datetime, str]] = [ + (date_by_sha[sha], sha) for sha, deg in in_degree.items() if deg == 0 + ] + heapq.heapify(ready) + order: dict[str, int] = {} + while ready: + _, sha = heapq.heappop(ready) + if sha in order: + continue + order[sha] = len(order) + 1 + for child in children.get(sha, ()): + in_degree[child] -= 1 + if in_degree[child] == 0: + heapq.heappush(ready, (date_by_sha[child], child)) + + # Fallback if synthetic metadata ever contains a cycle. + if len(order) < len(all_shas): + remaining = sorted( + ((date_by_sha[sha], sha) for sha in all_shas if sha not in order) + ) + for _, sha in remaining: + order[sha] = len(order) + 1 + return order + + +def build_commits(runs: list[dict]) -> list[dict]: + """Order commits across all nodes by committer_date, breaking ties by ancestry then sha.""" + order = _order_shas(runs) + rows: list[dict] = [] + for run in runs: + parents = list(run.get("parents") or []) + rows.append({ + "sha": run["sha"], + "short": short_sha(run["sha"]), + "parents": parents, + "parent": parents[0] if parents else None, + "author_date": run.get("author_date"), + "committer_date": run.get("committer_date"), + "subject": run.get("subject") or "", + "branch": run.get("branch") or "", + "node": run["node"], + "state": run.get("state") or "unknown", + "reason": run.get("reason"), + "started_at": run.get("started_at"), + "finished_at": run.get("finished_at"), + "suites_ok": int(run.get("suites_ok") or 0), + "suites_failed": int(run.get("suites_failed") or 0), + "results": int(run.get("results") or 0), + "commit_order": order[run["sha"]], + "has_log": bool(run.get("log")), + }) + rows.sort(key=lambda r: (-r["commit_order"], r["node"])) + return rows + + +def build_points(raw_points: list[dict], commits: list[dict]) -> list[dict]: + """Attach commit_order, committer_date, and parent to each point and sort.""" + by_pair = {(c["sha"], c["node"]): c for c in commits} + rows: list[dict] = [] + for pt in raw_points: + c = by_pair.get((pt["sha"], pt["node"])) + rows.append({ + "sha": pt["sha"], + "commit_order": c["commit_order"] if c else 0, + "committer_date": c["committer_date"] if c else pt.get("started_at"), + "parent": (c["parent"] or "") if c else "", + "node": pt["node"], + "suite": pt["suite"], + "case": pt["case"], + "params": pt["params"], + "quantity": pt["quantity"], + "unit": pt["unit"], + "direction": pt["direction"], + "status": pt["status"], + "reason": pt["reason"], + "estimate": pt["estimate"], + "mean": pt["mean"], + "min": pt["min"], + "max": pt["max"], + "n": int(pt["n"]), + "started_at": pt.get("started_at"), + "run_key": pt["run_key"], + "ingest_key": pt["ingest_key"], + }) + rows.sort(key=lambda r: (_series_key(r), r["commit_order"])) + return rows + + +def build_changes(points: list[dict]) -> list[dict]: + """Every measured point compared against the immediately preceding point in its series.""" + grouped: dict[tuple, list[dict]] = defaultdict(list) + for pt in points: + if is_measured(pt): + grouped[_series_key(pt)].append(pt) + + rows: list[dict] = [] + for key in sorted(grouped): + series_pts = sorted(grouped[key], key=lambda p: p["commit_order"]) + for prev, cur in zip(series_pts, series_pts[1:]): + prev_est = prev["estimate"] + rel = (cur["estimate"] / prev_est - 1.0) if prev_est > 0 else None + rows.append({ + "sha": cur["sha"], + "commit_order": cur["commit_order"], + "committer_date": cur["committer_date"], + "parent": cur["parent"], + "node": cur["node"], + "suite": cur["suite"], + "case": cur["case"], + "params": cur["params"], + "quantity": cur["quantity"], + "unit": cur["unit"], + "direction": cur["direction"], + "estimate": cur["estimate"], + "prev_estimate": prev_est, + "prev_sha": prev["sha"], + "prev_order": prev["commit_order"], + "rel_change": rel, + "prev_is_parent": prev["sha"] == cur["parent"], + "gap": cur["commit_order"] - prev["commit_order"], + }) + rows.sort(key=lambda r: (_series_key(r), r["commit_order"])) + return rows + + +def build_latest(points: list[dict]) -> list[dict]: + """One summary row per series that has at least one measured point.""" + grouped: dict[tuple, list[dict]] = defaultdict(list) + for pt in points: + if is_measured(pt): + grouped[_series_key(pt)].append(pt) + + rows: list[dict] = [] + for key in sorted(grouped): + pts_desc = sorted(grouped[key], key=lambda p: p["commit_order"], reverse=True) + newest = pts_desc[0] + hist = [p["estimate"] for p in pts_desc] + recent = hist[:RECENT] + diffs = [ + abs(recent[i] / recent[i + 1] - 1.0) + for i in range(len(recent) - 1) + if recent[i + 1] != 0 + ] + noise = float(statistics.median(diffs)) if len(diffs) >= 2 else None + if len(recent) > 1: + avg = sum(recent) / len(recent) + cv = float(statistics.pstdev(recent) / avg) if avg != 0 else None + else: + cv = None + + rows.append({ + "suite": newest["suite"], + "case": newest["case"], + "params": newest["params"], + "quantity": newest["quantity"], + "unit": newest["unit"], + "direction": newest["direction"], + "node": newest["node"], + "sha": newest["sha"], + "commit_order": newest["commit_order"], + "committer_date": newest["committer_date"], + "points": len(pts_desc), + "first_order": pts_desc[-1]["commit_order"], + "estimate": hist[0], + "vs_prev": (hist[0] / recent[1] - 1.0) if len(recent) > 1 and recent[1] != 0 else None, + "vs_10": (hist[0] / recent[10] - 1.0) if len(recent) > 10 and recent[10] != 0 else None, + "vs_100": (hist[0] / hist[100] - 1.0) if len(hist) > 100 and hist[100] != 0 else None, + "noise": noise, + "cv": cv, + "noise_rank": 0.0, + }) + + # Standard competition rank (SQL rank() OVER (PARTITION BY suite ORDER BY noise ASC NULLS LAST)). + by_suite: dict[str, list[dict]] = defaultdict(list) + for r in rows: + by_suite[r["suite"]].append(r) + for suite_rows in by_suite.values(): + denom = max(len(suite_rows) - 1, 1) + ordered = sorted(suite_rows, key=lambda r: (r["noise"] is None, r["noise"] if r["noise"] is not None else 0.0)) + prev_val = object() + rank = 0 + for idx, row in enumerate(ordered, start=1): + val = row["noise"] + if val != prev_val: + rank = idx + prev_val = val + row["noise_rank"] = (rank - 1) / denom + + rows.sort(key=_series_key) + return rows + + +def build_movers(changes: list[dict]) -> list[dict]: + """Top MOVERS_PER_COMMIT changes by |rel_change| per (sha, node).""" + by_pair: dict[tuple[str, str], list[dict]] = defaultdict(list) + for ch in changes: + if ch["rel_change"] is not None: + by_pair[(ch["sha"], ch["node"])].append(ch) + rows: list[dict] = [] + for group in by_pair.values(): + group.sort(key=lambda ch: abs(ch["rel_change"]), reverse=True) + rows.extend(group[:MOVERS_PER_COMMIT]) + rows.sort(key=lambda ch: (-ch["commit_order"], -abs(ch["rel_change"]))) + return rows + + +def build_marts( + runs: list[dict], + raw_points: list[dict], + *, + ingest_problems: int = 0, + source: str = "store", + has_parquet_marts: bool = False, +) -> dict: + """Compute all derived tables and manifest metadata in memory.""" + # Ensure every (sha, node) present in raw_points has a run row so orphan results + # (for example results.parquet without run.parquet) get a real commit_order and page. + known_pairs = {(r["sha"], r["node"]) for r in runs} + orphan_points = [pt for pt in raw_points if (pt["sha"], pt["node"]) not in known_pairs] + all_runs = list(runs) + if orphan_points: + all_runs.extend(synthesize_runs(orphan_points, default_state="unknown")) + + commits = build_commits(all_runs) + points = build_points(raw_points, commits) if raw_points else [] + changes = build_changes(points) if points else [] + latest = build_latest(points) if points else [] + movers = build_movers(changes) if changes else [] + + suite_counts = dict(sorted(Counter(p["suite"] for p in points).items())) + newest = commits[0] if commits else None + latest_commit = None + if newest is not None: + latest_commit = { + k: (v.isoformat() if isinstance(v, dt.datetime) else v) + for k, v in newest.items() + if k != "has_log" + } + + counts = { + "commits": len(commits), + "points": len(points), + "changes": len(changes), + "latest": len(latest), + "movers": len(movers), + "distinct_commits": len({c["sha"] for c in commits}), + "nodes": sorted({c["node"] for c in commits}), + "series": len(latest), + "ingest_problems": int(ingest_problems), + } + tables = ( + { + "commits": "commits.parquet", + "points": "points.parquet", + "changes": "changes.parquet", + "latest": "latest.parquet", + "movers": "movers.parquet", + } + if has_parquet_marts + else {} + ) + manifest = { + "built_at": dt.datetime.now(UTC).isoformat(timespec="seconds"), + "source": source, + "counts": counts, + "latest_commit": latest_commit, + "suites": suite_counts, + "tables": tables, + "notes": { + "commit_order": "rank of committer_date (breaking ties by ancestry then sha) among measured commits", + "rel_change": "estimate / previous measured point of the same series - 1; a heuristic, not a verdict", + "noise": f"median of |consecutive rel_change| over the last {RECENT} points of the series; NULL below 3 points", + }, + } + return { + "commits": commits, + "points": points, + "changes": changes, + "latest": latest, + "movers": movers, + "manifest": manifest, + } + + +def _mart_schemas(): + pa, _ = _require_pyarrow("writing Parquet marts") + ts = pa.timestamp("us", tz="UTC") + return { + "commits": pa.schema([ + ("sha", pa.string()), ("short", pa.string()), ("parents", pa.list_(pa.string())), ("parent", pa.string()), + ("author_date", ts), ("committer_date", ts), ("subject", pa.string()), ("branch", pa.string()), + ("node", pa.string()), ("state", pa.string()), ("reason", pa.string()), ("started_at", ts), + ("finished_at", ts), ("suites_ok", pa.int32()), ("suites_failed", pa.int32()), ("results", pa.int32()), + ("commit_order", pa.int32()), + ]), + "points": pa.schema([ + ("sha", pa.string()), ("commit_order", pa.int32()), ("committer_date", ts), ("parent", pa.string()), + ("node", pa.string()), ("suite", pa.string()), ("case", pa.string()), ("params", pa.string()), + ("quantity", pa.string()), ("unit", pa.string()), ("direction", pa.string()), ("status", pa.string()), + ("reason", pa.string()), ("estimate", pa.float64()), ("mean", pa.float64()), ("min", pa.float64()), + ("max", pa.float64()), ("n", pa.int32()), ("started_at", ts), ("run_key", pa.string()), + ("ingest_key", pa.string()), + ]), + "changes": pa.schema([ + ("sha", pa.string()), ("commit_order", pa.int32()), ("committer_date", ts), ("parent", pa.string()), + ("node", pa.string()), ("suite", pa.string()), ("case", pa.string()), ("params", pa.string()), + ("quantity", pa.string()), ("unit", pa.string()), ("direction", pa.string()), ("estimate", pa.float64()), + ("prev_estimate", pa.float64()), ("prev_sha", pa.string()), ("prev_order", pa.int32()), + ("rel_change", pa.float64()), ("prev_is_parent", pa.bool_()), ("gap", pa.int32()), + ]), + "latest": pa.schema([ + ("suite", pa.string()), ("case", pa.string()), ("params", pa.string()), ("quantity", pa.string()), + ("unit", pa.string()), ("direction", pa.string()), ("node", pa.string()), ("sha", pa.string()), + ("commit_order", pa.int32()), ("committer_date", ts), ("points", pa.int32()), ("first_order", pa.int32()), + ("estimate", pa.float64()), ("vs_prev", pa.float64()), ("vs_10", pa.float64()), ("vs_100", pa.float64()), + ("noise", pa.float64()), ("cv", pa.float64()), ("noise_rank", pa.float64()), + ]), + } + + +def write_parquet_marts(marts: dict, data_dir: Path) -> None: + """Write the five derived Parquet marts under `data_dir` with small row groups.""" + pa, pq = _require_pyarrow("writing Parquet marts") + data_dir = Path(data_dir) + data_dir.mkdir(parents=True, exist_ok=True) + schemas = _mart_schemas() + schemas["movers"] = schemas["changes"] + + for name in ("commits", "points", "changes", "latest", "movers"): + rows = marts[name] + if name == "commits": + rows = [{k: v for k, v in r.items() if k != "has_log"} for r in rows] + table = pa.Table.from_pylist(rows, schema=schemas[name]) if rows else pa.Table.from_pylist([], schema=schemas[name]) + pq.write_table( + table, + data_dir / f"{name}.parquet", + compression="zstd", + row_group_size=2048, + write_statistics=True, + ) diff --git a/src/benchx/views/payloads.py b/src/benchx/views/payloads.py new file mode 100644 index 00000000..d6c9769a --- /dev/null +++ b/src/benchx/views/payloads.py @@ -0,0 +1,558 @@ +"""Derive the static dashboard's JSON view payloads from the in-memory marts. + +No query engine is required; every number shown by the page is computed here in +pure Python and written as small JSON documents under `data_dir / "views"`: + + landing.json stats, each node's newest commit run (movers & failures), + most recently finished run, recent commits, noisiest series, + suite wall time + series.json one entry per series (including never-succeeded series) + sparks.json every series' last SPARK_POINTS points + shared commit axis + history/.json every series of one suite as aligned parallel arrays + commits/.json one commit: its nodes' status, failures, and changes +""" + +from __future__ import annotations + +import datetime as dt +import json +from collections import defaultdict +from pathlib import Path + +from .inputs import UTC, ViewError, safe_path_segment, short_sha +from .marts import is_failed_status, is_measured + +NOTABLE = 0.05 +RECENT_COMMITS = 30 +MOVERS = 15 +NOISIEST = 10 +SPARK_POINTS = 60 + + +def _iso(value) -> str | None: + if value is None: + return None + if isinstance(value, dt.datetime): + return value.astimezone(UTC).isoformat() if value.tzinfo else value.replace(tzinfo=UTC).isoformat() + return str(value) + + +def _safe_child(parent: Path, stem: str, suffix: str = ".json") -> Path: + base = parent.resolve() + target = (base / f"{safe_path_segment(stem)}{suffix}").resolve() + if not target.is_relative_to(base): + raise ViewError(f"refusing to write outside {base}: {stem!r}") + return target + + +def _dump(doc, path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(doc, separators=(",", ":"), ensure_ascii=False) + "\n", encoding="utf-8") + + +def signed(direction: str, rel_change) -> str | None: + """'regressed' or 'improved' for a change, by the quantity's direction.""" + if rel_change is None: + return None + worse = rel_change > 0 if direction != "higher-is-better" else rel_change < 0 + return "regressed" if worse else "improved" + + +def series_key(r: dict) -> tuple: + return (r["suite"], r["case"], r["params"], r["quantity"], r["node"]) + + +def newest_commits(commits: list[dict]) -> dict[str, dict]: + """Per node, its run of the newest commit (commit order, not execution time).""" + out: dict[str, dict] = {} + for c in commits: # already sorted by commit_order descending + out.setdefault(c["node"], c) + return out + + +def last_executed(commits: list[dict]) -> dict[str, dict]: + """Per node, the run that finished (or started) most recently.""" + out: dict[str, dict] = {} + for c in commits: + when = c.get("finished_at") or c.get("started_at") or "" + prev = out.get(c["node"]) + prev_when = (prev.get("finished_at") or prev.get("started_at") or "") if prev else "" + if prev is None or when > prev_when: + out[c["node"]] = c + return out + + +def _serialize_commit(c: dict, notable_entry: dict, failed_count: int) -> dict: + return { + "sha": c["sha"], + "short": c["short"], + "parent": c["parent"], + "subject": c["subject"], + "committer_date": _iso(c["committer_date"]), + "node": c["node"], + "state": c["state"], + "reason": c["reason"], + "started_at": _iso(c["started_at"]), + "finished_at": _iso(c["finished_at"]), + "suites_ok": c["suites_ok"], + "suites_failed": c["suites_failed"], + "results": c["results"], + "commit_order": c["commit_order"], + "has_log": bool(c.get("has_log", False)), + "regressed": notable_entry.get("regressed", 0), + "improved": notable_entry.get("improved", 0), + "compared": notable_entry.get("compared", 0), + "failed_results": failed_count, + } + + +def _write_commit_pages( + out: Path, + commits: list[dict], + changes_by_sha: dict[str, list[dict]], + failures_by_sha: dict[str, list[dict]], +) -> int: + commits_dir = out / "commits" + if commits_dir.is_dir(): + for stale in commits_dir.glob("*.json"): + stale.unlink() + by_sha: dict[str, list[dict]] = {} + for c in commits: + by_sha.setdefault(c["sha"], []).append(c) + for sha, nodes_of in by_sha.items(): + head = nodes_of[0] + _dump( + { + "sha": sha, + "short": head["short"], + "subject": head["subject"], + "committer_date": head["committer_date"], + "parent": head["parent"], + "commit_order": head["commit_order"], + "nodes": nodes_of, + "failures": failures_by_sha.get(sha, []), + "changes": changes_by_sha.get(sha, []), + }, + _safe_child(commits_dir, sha), + ) + return len(by_sha) + + +def _build_empty(out: Path, commits: list[dict]) -> dict: + history_dir = out / "history" + if history_dir.is_dir(): + for stale in history_dir.glob("*.json"): + stale.unlink() + newest_by_node = newest_commits(commits) + _dump([], out / "series.json") + _dump({"axis": [], "series": []}, out / "sparks.json") + _dump( + { + "built_at": dt.datetime.now(UTC).isoformat(timespec="seconds"), + "notable": NOTABLE, + "stats": { + "commits": len({c["sha"] for c in commits}), + "results": 0, + "series": 0, + "nodes": sorted(newest_by_node), + "latest": commits[0] if commits else None, + }, + "latest_runs": list(newest_by_node.values()), + "last_executed": last_executed(commits), + "recent_commits": commits[:RECENT_COMMITS], + "movers": {"regressed": [], "improved": []}, + "failures": [], + "noisiest": [], + "suite_time": [], + }, + out / "landing.json", + ) + return { + "series": 0, + "recent_commits": min(len(commits), RECENT_COMMITS), + "sparks": 0, + "history_suites": 0, + "commit_pages": _write_commit_pages(out, commits, {}, {}), + } + + +def build_views(marts: dict, data_dir: Path) -> dict: + """Write every JSON view payload under `data_dir / 'views'` and return counts.""" + out = Path(data_dir).resolve() / "views" + commits_raw = marts["commits"] + points = marts["points"] + changes = marts["changes"] + latest_raw = marts["latest"] + has_points = len(points) > 0 + + notable: dict[tuple[str, str], dict] = {} + failed_results: dict[tuple[str, str], int] = defaultdict(int) + if has_points: + for ch in changes: + pair = (ch["sha"], ch["node"]) + entry = notable.setdefault(pair, {"regressed": 0, "improved": 0, "compared": 0}) + entry["compared"] += 1 + rel = ch["rel_change"] + if rel is not None and abs(rel) > NOTABLE: + kind = signed(ch["direction"], rel) + if kind == "regressed": + entry["regressed"] += 1 + elif kind == "improved": + entry["improved"] += 1 + for pt in points: + if is_failed_status(pt["status"]): + failed_results[(pt["sha"], pt["node"])] += 1 + + commits = [ + _serialize_commit( + c, + notable.get((c["sha"], c["node"]), {}), + failed_results.get((c["sha"], c["node"]), 0), + ) + for c in commits_raw + ] + if not has_points: + return _build_empty(out, commits) + + counts: dict[str, int] = {} + + # Latest attempt of every series (measured, skipped, or failed). + attempts: dict[tuple, dict] = {} + for pt in points: + key = series_key(pt) + prev = attempts.get(key) + if prev is None or pt["commit_order"] >= prev["attempt_order"]: + attempts[key] = { + "suite": pt["suite"], + "case": pt["case"], + "params": pt["params"], + "quantity": pt["quantity"], + "node": pt["node"], + "unit": pt["unit"], + "direction": pt["direction"], + "status": pt["status"], + "reason": pt["reason"], + "attempt_order": pt["commit_order"], + "attempt_sha": pt["sha"], + } + + latest_by_key: dict[tuple, dict] = {} + series_list: list[dict] = [] + for r in latest_raw: + key = series_key(r) + a = attempts[key] + entry = { + "suite": r["suite"], + "case": r["case"], + "params": r["params"], + "quantity": r["quantity"], + "unit": r["unit"], + "direction": r["direction"], + "node": r["node"], + "sha": r["sha"], + "commit_order": r["commit_order"], + "first_order": r["first_order"], + "committer_date": _iso(r["committer_date"]), + "points": r["points"], + "estimate": r["estimate"], + "vs_prev": r["vs_prev"], + "vs_10": r["vs_10"], + "vs_100": r["vs_100"], + "noise": r["noise"], + "cv": r["cv"], + "noise_rank": r["noise_rank"], + "status": a["status"], + "reason": a["reason"], + "attempt_order": a["attempt_order"], + "attempt_sha": a["attempt_sha"], + } + latest_by_key[key] = entry + series_list.append(entry) + + for key, a in attempts.items(): + if key not in latest_by_key: + entry = { + **a, + "sha": a["attempt_sha"], + "commit_order": a["attempt_order"], + "first_order": None, + "committer_date": None, + "points": 0, + "estimate": None, + "vs_prev": None, + "vs_10": None, + "vs_100": None, + "noise": None, + "cv": None, + "noise_rank": None, + } + latest_by_key[key] = entry + series_list.append(entry) + + series_list.sort(key=series_key) + counts["series"] = len(series_list) + _dump(series_list, out / "series.json") + + # Movers on each node's latest commit with changes. + top_change_order: dict[str, int] = {} + for ch in changes: + node = ch["node"] + if node not in top_change_order or ch["commit_order"] > top_change_order[node]: + top_change_order[node] = ch["commit_order"] + + top_movers: list[dict] = [] + for ch in changes: + if ch["rel_change"] is None or ch["commit_order"] != top_change_order.get(ch["node"]): + continue + lt = latest_by_key.get(series_key(ch)) + top_movers.append({ + "suite": ch["suite"], + "case": ch["case"], + "params": ch["params"], + "quantity": ch["quantity"], + "unit": ch["unit"], + "direction": ch["direction"], + "node": ch["node"], + "sha": ch["sha"], + "short": short_sha(ch["sha"]), + "estimate": ch["estimate"], + "prev_estimate": ch["prev_estimate"], + "rel_change": ch["rel_change"], + "prev_sha": ch["prev_sha"], + "prev_is_parent": ch["prev_is_parent"], + "gap": ch["gap"], + "noise": lt["noise"] if lt else None, + "points": lt["points"] if lt else None, + }) + top_movers.sort(key=lambda m: abs(m["rel_change"]), reverse=True) + + regressed: list[dict] = [] + improved: list[dict] = [] + per_node: dict[tuple[str, str | None], int] = defaultdict(int) + for m in top_movers: + kind = signed(m["direction"], m["rel_change"]) + per_node[(m["node"], kind)] += 1 + if per_node[(m["node"], kind)] <= MOVERS: + (regressed if kind == "regressed" else improved).append(m) + + # Failures on each node's newest commit. + newest_by_node = newest_commits(commits) + failures: list[dict] = [] + for key in sorted(attempts): + a = attempts[key] + top = newest_by_node.get(key[4]) + if top and a["attempt_sha"] == top["sha"] and is_failed_status(a["status"]): + last_ok = latest_by_key[key] + failures.append({ + "suite": key[0], + "case": key[1], + "params": key[2], + "quantity": key[3], + "node": key[4], + "status": a["status"], + "reason": a["reason"], + "sha": a["attempt_sha"], + "last_success_sha": last_ok["sha"] if last_ok.get("points") else None, + }) + + noisiest = sorted( + ( + { + "suite": r["suite"], + "case": r["case"], + "params": r["params"], + "quantity": r["quantity"], + "node": r["node"], + "noise": r["noise"], + "cv": r["cv"], + "points": r["points"], + "noise_rank": r["noise_rank"], + } + for r in latest_raw + if r["noise"] is not None and r["points"] >= 3 + ), + key=lambda r: r["noise"], + reverse=True, + )[:NOISIEST] + + # Suite wall time on each node's latest commit with points. + top_point_order: dict[str, int] = {} + for pt in points: + node = pt["node"] + if node not in top_point_order or pt["commit_order"] > top_point_order[node]: + top_point_order[node] = pt["commit_order"] + + suite_groups: dict[tuple[str, str, str], list[dict]] = defaultdict(list) + for pt in points: + if pt["commit_order"] == top_point_order.get(pt["node"]): + suite_groups[(pt["suite"], pt["node"], pt["sha"])].append(pt) + suite_time = [] + for (suite, node, sha), pts in suite_groups.items(): + times = [p["started_at"] for p in pts if p["started_at"] is not None] + seconds = int((max(times) - min(times)).total_seconds()) if len(times) >= 2 else 0 + suite_time.append({ + "suite": suite, + "node": node, + "sha": sha, + "short": short_sha(sha), + "cases": len(pts), + "failed": sum(1 for p in pts if is_failed_status(p["status"])), + "seconds": seconds, + }) + suite_time.sort(key=lambda r: (-r["seconds"], r["suite"], r["node"])) + + nodes = sorted({c["node"] for c in commits}) + landing = { + "built_at": dt.datetime.now(UTC).isoformat(timespec="seconds"), + "notable": NOTABLE, + "stats": { + "commits": len({p["sha"] for p in points}), + "results": len(points), + "series": len(series_list), + "nodes": nodes, + "latest": commits[0] if commits else None, + }, + "latest_runs": list(newest_by_node.values()), + "last_executed": last_executed(commits), + "recent_commits": commits[:RECENT_COMMITS], + "movers": {"regressed": regressed, "improved": improved}, + "failures": failures, + "noisiest": noisiest, + "suite_time": suite_time, + } + _dump(landing, out / "landing.json") + counts["recent_commits"] = len(landing["recent_commits"]) + + # Sparks: last SPARK_POINTS commits and per-series measured points. + axis_by_order: dict[int, dict] = {} + for pt in points: + order = pt["commit_order"] + if order not in axis_by_order: + axis_by_order[order] = { + "commit_order": order, + "sha": pt["sha"], + "short": short_sha(pt["sha"]), + "date": _iso(pt["committer_date"]), + } + spark_axis = [axis_by_order[o] for o in sorted(axis_by_order)[-SPARK_POINTS:]] + + ok_by_series: dict[tuple, list[dict]] = defaultdict(list) + for pt in points: + if is_measured(pt): + ok_by_series[series_key(pt)].append(pt) + + spark_series = [] + for key in sorted(ok_by_series): + pts = sorted(ok_by_series[key], key=lambda p: p["commit_order"])[-SPARK_POINTS:] + lt = latest_by_key.get(key, {}) + a = attempts[key] + head = pts[-1] + spark_series.append({ + "suite": head["suite"], + "case": head["case"], + "params": head["params"], + "quantity": head["quantity"], + "unit": head["unit"], + "direction": head["direction"], + "node": head["node"], + "latest": lt.get("estimate"), + "vs_prev": lt.get("vs_prev"), + "noise": lt.get("noise"), + "points": lt.get("points"), + "sha": lt.get("sha"), + "orders": [p["commit_order"] for p in pts], + "estimates": [p["estimate"] for p in pts], + "status": a["status"], + "reason": a["reason"], + }) + _dump({"axis": spark_axis, "series": spark_series}, out / "sparks.json") + counts["sparks"] = len(spark_series) + + # History per suite: parallel arrays per series + commit axis. + history_dir = out / "history" + if history_dir.is_dir(): + for stale in history_dir.glob("*.json"): + stale.unlink() + + subject_by_pair = {(c["sha"], c["node"]): c["subject"] for c in commits} + points_by_suite: dict[str, list[dict]] = defaultdict(list) + for pt in points: + points_by_suite[pt["suite"]].append(pt) + + for suite in sorted(points_by_suite): + suite_pts = points_by_suite[suite] + axis_map: dict[tuple[int, str], dict] = {} + series_groups: dict[tuple, list[dict]] = defaultdict(list) + for pt in suite_pts: + axis_key = (pt["commit_order"], pt["sha"]) + if axis_key not in axis_map: + axis_map[axis_key] = { + "commit_order": pt["commit_order"], + "sha": pt["sha"], + "short": short_sha(pt["sha"]), + "date": _iso(pt["committer_date"]), + "subject": subject_by_pair.get((pt["sha"], pt["node"]), ""), + } + series_groups[(pt["case"], pt["params"], pt["quantity"], pt["node"])].append(pt) + + axis = [axis_map[k] for k in sorted(axis_map)] + series_map: dict[str, dict] = {} + for skey in sorted(series_groups): + pts = sorted(series_groups[skey], key=lambda p: p["commit_order"]) + head = pts[0] + series_map["|".join(skey)] = { + "case": head["case"], + "params": head["params"], + "quantity": head["quantity"], + "unit": head["unit"], + "direction": head["direction"], + "node": head["node"], + "orders": [p["commit_order"] for p in pts], + "estimates": [p["estimate"] for p in pts], + "mins": [p["min"] for p in pts], + "maxs": [p["max"] for p in pts], + "ns": [p["n"] for p in pts], + "statuses": [p["status"] for p in pts], + "reasons": [p["reason"] for p in pts], + } + _dump({"suite": suite, "commits": axis, "series": series_map}, _safe_child(history_dir, suite)) + counts["history_suites"] = len(points_by_suite) + + # Per-commit pages: nodes, failures, and every change (largest first). + changes_by_sha: dict[str, list[dict]] = defaultdict(list) + for ch in changes: + lt = latest_by_key.get(series_key(ch)) + changes_by_sha[ch["sha"]].append({ + "suite": ch["suite"], + "case": ch["case"], + "params": ch["params"], + "quantity": ch["quantity"], + "unit": ch["unit"], + "direction": ch["direction"], + "node": ch["node"], + "estimate": ch["estimate"], + "prev_estimate": ch["prev_estimate"], + "rel_change": ch["rel_change"], + "prev_sha": ch["prev_sha"], + "prev_is_parent": ch["prev_is_parent"], + "gap": ch["gap"], + "noise": lt["noise"] if lt else None, + }) + for sha_changes in changes_by_sha.values(): + sha_changes.sort(key=lambda ch: abs(ch["rel_change"]) if ch["rel_change"] is not None else -1.0, reverse=True) + + failures_by_sha: dict[str, list[dict]] = defaultdict(list) + for pt in points: + if is_failed_status(pt["status"]): + failures_by_sha[pt["sha"]].append({ + "suite": pt["suite"], + "case": pt["case"], + "params": pt["params"], + "quantity": pt["quantity"], + "node": pt["node"], + "status": pt["status"], + "reason": pt["reason"], + }) + + counts["commit_pages"] = _write_commit_pages(out, commits, changes_by_sha, failures_by_sha) + return counts diff --git a/src/benchx/views/site.py b/src/benchx/views/site.py new file mode 100644 index 00000000..f05258bb --- /dev/null +++ b/src/benchx/views/site.py @@ -0,0 +1,233 @@ +"""Assemble a static benchx dashboard directory and serve it locally over HTTP.""" + +from __future__ import annotations + +import errno +import functools +import http.server +import importlib.util +import json +import re +import shutil +import sys +import webbrowser +from pathlib import Path +from urllib.parse import quote + +from . import inputs, marts as marts_mod, payloads + +STATIC_DIR = Path(__file__).resolve().parent / "static" +STATIC_FILES = ("app.js", "style.css", "favicon.svg", "logo.svg") +_STAMP_RE = re.compile(r'(href="(style\.css|favicon\.svg)|src="(app\.js))"') + + +def _has_pyarrow() -> bool: + if "pyarrow" in sys.modules: + return sys.modules["pyarrow"] is not None + try: + return importlib.util.find_spec("pyarrow") is not None + except (ValueError, ModuleNotFoundError): + return False + + +def _github_commit_url(source_uri: str | None) -> str | None: + if not source_uri: + return None + uri = source_uri.strip() + if uri.startswith("git@github.com:"): + uri = "https://github.com/" + uri.removeprefix("git@github.com:") + if uri.startswith("https://github.com/") or uri.startswith("http://github.com/"): + base = uri.removesuffix(".git").rstrip("/") + return f"{base}/commit/{{sha}}" + return None + + +def resolve_project( + *, + project_file: Path | None = None, + title: str | None = None, + config=None, +) -> dict: + """Build the `project.json` metadata copied beside the view payloads.""" + if project_file is not None: + path = Path(project_file) + if not path.is_file(): + raise inputs.ViewError(f"project file not found: {path}") + try: + project = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + raise inputs.ViewError(f"{path}: invalid JSON: {exc}") from exc + if not isinstance(project, dict): + raise inputs.ViewError(f"{path}: expected a JSON object") + else: + name = config.data.get("project") if config is not None else None + source_uri = config.section("source").get("uri") if config is not None else None + project = { + "name": name or "benchmarks", + "title": f"{name} benchmarks" if name else "Benchmarks", + "description": "Benchmarks measured with benchx.", + } + if source_uri: + upstream = {"url": source_uri} + if commit_url := _github_commit_url(source_uri): + upstream["commit_url"] = commit_url + project["upstream"] = upstream + + if title: + project = {**project, "title": title} + return project + + +def _write_static(out: Path, stamp: str) -> None: + for name in STATIC_FILES: + shutil.copy2(STATIC_DIR / name, out / name) + index = (STATIC_DIR / "index.html").read_text(encoding="utf-8") + if stamp: + encoded = quote(stamp, safe="") + index = _STAMP_RE.sub(rf'\1?v={encoded}"', index) + (out / "index.html").write_text(index, encoding="utf-8") + + +def _write_logs(runs: list[dict], data_dir: Path) -> int: + logs_root = (Path(data_dir) / "logs").resolve() + if logs_root.is_dir(): + shutil.rmtree(logs_root) + written = 0 + for run in runs: + log = run.get("log") or "" + if not log: + continue + sha_seg = inputs.safe_path_segment(run["sha"]) + node_seg = inputs.safe_path_segment(run["node"]) + log_path = (logs_root / sha_seg / f"{node_seg}.txt").resolve() + if not log_path.is_relative_to(logs_root): + raise inputs.ViewError(f"refusing to write log outside {logs_root}: {run['sha']!r}/{run['node']!r}") + log_path.parent.mkdir(parents=True, exist_ok=True) + log_path.write_text(log, encoding="utf-8") + written += 1 + return written + + +def build_site( + out: Path, + *, + store_path: str | Path | None = None, + results_paths: list[str | Path] | None = None, + runs_dir: str | Path | None = None, + project_file: Path | None = None, + title: str | None = None, + repo: Path | None = None, + write_marts: bool | None = None, + stamp: str | None = None, + config=None, +) -> dict: + """Build a complete static benchx dashboard under `out` and return a summary.""" + chosen = sum(x is not None for x in (store_path, results_paths, runs_dir)) + if chosen != 1: + raise inputs.ViewError("give exactly one of store_path, results_paths, or runs_dir") + + # Validate project metadata and optional PyArrow requirements before touching `out`. + project = resolve_project(project_file=project_file, title=title, config=config) + if write_marts is None: + emit_marts = _has_pyarrow() + else: + emit_marts = bool(write_marts) + if emit_marts: + inputs._require_pyarrow("writing Parquet marts") + + outcome = None + if runs_dir is not None: + source = "runs" + runs, raw_points, ingest_problems = inputs.read_runs_dir(runs_dir) + elif results_paths is not None: + source = "results" + runs, raw_points, ingest_problems, outcome = inputs.read_results( + list(results_paths), repo=repo + ) + else: + source = "store" + runs, raw_points, ingest_problems = inputs.read_store(store_path, repo=repo) + + marts = marts_mod.build_marts( + runs, + raw_points, + ingest_problems=ingest_problems, + source=source, + has_parquet_marts=emit_marts, + ) + + out = Path(out).resolve() + data_dir = out / "data" + data_dir.mkdir(parents=True, exist_ok=True) + + if emit_marts: + marts_mod.write_parquet_marts(marts, data_dir) + else: + for name in ("commits", "points", "changes", "latest", "movers"): + stale = data_dir / f"{name}.parquet" + if stale.exists(): + stale.unlink() + + logs_written = _write_logs(runs, data_dir) + (data_dir / "project.json").write_text(json.dumps(project, indent=1) + "\n", encoding="utf-8") + (data_dir / "manifest.json").write_text(json.dumps(marts["manifest"], indent=1) + "\n", encoding="utf-8") + view_counts = payloads.build_views(marts, data_dir) + _write_static(out, stamp if stamp is not None else marts["manifest"]["built_at"]) + + return { + "out": str(out), + "source": source, + "counts": marts["manifest"]["counts"], + "views": view_counts, + "marts": emit_marts, + "logs": logs_written, + "outcome": outcome, + } + + +class _QuietHandler(http.server.SimpleHTTPRequestHandler): + def log_message(self, format, *args): # noqa: A002 + return + + +def create_server( + directory: Path, + *, + host: str = "127.0.0.1", + port: int = 8000, + fallback_port: bool = False, +) -> tuple[http.server.ThreadingHTTPServer, str]: + """Bind a local HTTP server for `directory` without starting its loop.""" + handler = functools.partial(_QuietHandler, directory=str(Path(directory).resolve())) + try: + server = http.server.ThreadingHTTPServer((host, port), handler) + except OSError as exc: + if fallback_port and exc.errno == errno.EADDRINUSE and port != 0: + server = http.server.ThreadingHTTPServer((host, 0), handler) + else: + raise + bound_host, bound_port = server.server_address[:2] + display_host = "127.0.0.1" if bound_host in ("", "0.0.0.0") else bound_host + return server, f"http://{display_host}:{bound_port}/" + + +def serve_site( + directory: Path, + *, + host: str = "127.0.0.1", + port: int = 8000, + fallback_port: bool = False, + open_browser: bool = False, + announce=print, +) -> None: + """Serve `directory` over local HTTP until interrupted with Ctrl-C.""" + server, url = create_server(directory, host=host, port=port, fallback_port=fallback_port) + announce(f"serving {Path(directory).resolve()} at {url} (Ctrl-C to stop)") + if open_browser: + webbrowser.open(url) + try: + server.serve_forever() + except KeyboardInterrupt: + pass + finally: + server.server_close() diff --git a/src/benchx/views/static/app.js b/src/benchx/views/static/app.js new file mode 100644 index 00000000..04307446 --- /dev/null +++ b/src/benchx/views/static/app.js @@ -0,0 +1,880 @@ +// The benchx dashboard: a static page over the JSON views written by `bx view`. +// No query engine runs here; every number shown was computed ahead of time. +// Routes: #/ overview, #/suites, #/s/, #/b//?params=&q=&node=, +// #/c/, #/about. Names, links and the upstream come from data/project.json. +// Data is fetched with cache-busting because static CDNs may cache for hours. + +const DATA = "data/"; +const SERIES_VARS = ["--series-1", "--series-2", "--series-3", "--series-4", "--series-5", "--series-6", "--series-7", "--series-8"]; +const PAGE = 48; +// The benchx mark (logo.svg), inline so the legs follow the page's ink in +// both themes; the small variant (favicon.svg) is drawn heavier for 16-32 px. +const MARK = ''; +const MARK_SMALL = ''; +function wordmark(small) { return h("span", { class: "benchx" }, h("span", { html: small ? MARK_SMALL : MARK }), h("span", { class: "wm" }, "bench", h("b", {}, "x"))); } +const state = { v: "", project: null, manifest: null, landing: null, series: null, sparks: null, histories: new Map(), commits: new Map(), charts: [], cleanup: [], nodeSlots: new Map() }; + +// ---- utilities ------------------------------------------------------------------ + +function h(tag, attrs = {}, ...children) { + const node = document.createElement(tag); + for (const [k, v] of Object.entries(attrs)) { + if (v == null || v === false) continue; + if (k === "class") node.className = v; + else if (k === "html") node.innerHTML = v; // only for trusted markup written in this file + else if (k.startsWith("on")) node.addEventListener(k.slice(2), v); + else node.setAttribute(k, v === true ? "" : v); + } + for (const c of children.flat()) { + if (c == null || c === false) continue; + node.append(c.nodeType ? c : document.createTextNode(String(c))); + } + return node; +} + +function joinParts(parts, sep = " · ") { + const out = []; + for (const p of parts) { + if (p == null || p === false || p === "") continue; + if (out.length) out.push(sep); + out.push(p); + } + return out; +} + +async function getJSON(path, { fresh = false } = {}) { + const url = DATA + path + (fresh ? `?t=${Date.now()}` : `?v=${encodeURIComponent(state.v)}`); + const r = await fetch(url); + if (!r.ok) throw new Error(`${path}: HTTP ${r.status}`); + return r.json(); +} + +const UNITS = { + s: [["ns", 1e-9], ["µs", 1e-6], ["ms", 1e-3], ["s", 1]], + B: [["B", 1], ["KiB", 1024], ["MiB", 1024 ** 2], ["GiB", 1024 ** 3]], +}; +function pickScale(values, unit) { + const table = UNITS[unit]; + const m = Math.max(...values.filter((v) => v != null && isFinite(v)).map(Math.abs), 0); + if (!table || !m) return [unit, 1]; + let best = table[0]; + for (const t of table) if (m >= t[1]) best = t; + return best; +} +function fmtNum(v, digits = 3) { + if (v == null || !isFinite(v)) return "–"; + const a = Math.abs(v); + if (a >= 1000) return v.toLocaleString(undefined, { maximumFractionDigits: 0 }); + if (a >= 100) return v.toFixed(digits - 3 < 0 ? 0 : digits - 3); + if (a >= 10) return v.toFixed(digits - 2); + return v.toPrecision(digits); +} +function fmtValue(v, unit, scale) { + if (v == null) return "–"; + const [label, factor] = scale || pickScale([v], unit); + return `${fmtNum(v / factor)} ${label}`; +} +function fmtPct(x, digits = 1) { + if (x == null || !isFinite(x)) return "–"; + const s = (x * 100).toFixed(digits); + return (x > 0 ? "+" : "") + s + "%"; +} +function fmtAbsPct(x, digits = 1) { return x == null || !isFinite(x) ? "–" : (Math.abs(x) * 100).toFixed(digits) + "%"; } +function parseDate(s) { + if (!s) return null; + const iso = s.includes("T") ? s : s.replace(" ", "T") + (s.endsWith("Z") || /[+-]\d\d:\d\d$/.test(s) ? "" : "Z"); + const d = new Date(iso); + return isNaN(d) ? null : d; +} +function fmtDate(s) { + const d = parseDate(s); + return d ? d.toLocaleString(undefined, { month: "short", day: "numeric", hour: "2-digit", minute: "2-digit" }) : "–"; +} +function ago(s) { + const d = parseDate(s); + if (!d) return ""; + const m = Math.round((Date.now() - d) / 60000); + if (m < 60) return `${m} min ago`; + if (m < 48 * 60) return `${Math.round(m / 60)} h ago`; + return `${Math.round(m / 1440)} d ago`; +} +function shortSuite(s) { + let out = s.replace(/-benchmark$/g, ""); + const prefix = state.project?.name ? `${state.project.name}-` : ""; + if (prefix && out.startsWith(prefix)) out = out.slice(prefix.length); + return out; +} +function safeFile(s) { + const cleaned = String(s).replace(/[/\\\x00:*?"<>|]+/g, "_").replace(/\.\.+/g, "_"); + return cleaned && cleaned !== "." ? cleaned : "_"; +} +function shortSha(sha, len = 12) { + if (!sha) return ""; + if (sha.includes("-dirty-")) { + const [base, tree] = sha.split("-dirty-", 2); + return `${base.slice(0, 7)}+dirty-${tree.slice(0, 4)}`; + } + return sha.slice(0, len); +} +function isFailed(status) { return Boolean(status && !["success", "partial", "skipped"].includes(status)); } +function upstreamCommit(sha) { + if (!sha || sha.includes("-dirty-")) return null; + const tpl = state.project?.upstream?.commit_url; + return tpl ? tpl.replace("{sha}", sha) : null; +} +function runFiles(sha, node) { + if (state.manifest?.source !== "runs") return null; + const repo = state.project?.repository; + return repo ? `${repo}/tree/main/runs/${sha}/${node}` : null; +} +function runLog(sha, node, hasLog = true) { + return hasLog === false ? null : `data/logs/${encodeURIComponent(safeFile(sha))}/${encodeURIComponent(safeFile(node))}.txt`; +} +function multiNode() { return (state.landing?.stats.nodes || []).length > 1; } + +// A change is judged three ways: its direction (the quantity says whether up is +// worse), its size (under 0.5 % is flat), and its size against the series' noise +// (under twice the noise it is shown muted: observed, but not distinguishable +// from run-to-run variation). None of it is a verdict; see the about page. +const FLAT = 0.005; +function verdict(rel, direction) { + if (rel == null || !isFinite(rel)) return "none"; + if (Math.abs(rel) < FLAT) return "flat"; + const worse = direction === "higher-is-better" ? rel < 0 : rel > 0; + return worse ? "regressed" : "improved"; +} +function changeWords(direction, unit) { + if (unit === "s" && direction !== "higher-is-better") return { worse: "slower", better: "faster" }; + return { worse: "worse", better: "better" }; +} +function withinNoise(rel, noise) { return noise != null && rel != null && Math.abs(rel) < 2 * noise; } +// delta(rel, direction, {noise, unit}): noise undefined means "not known here", +// null means "the series has too few points for one". +function delta(rel, direction, opts = {}) { + const v = verdict(rel, direction); + const real = v === "regressed" || v === "improved"; + const within = real && withinNoise(rel, opts.noise); + const w = changeWords(direction, opts.unit); + const glyph = v === "none" ? "" : v === "flat" ? "■" : rel > 0 ? "▲" : "▼"; + const word = { regressed: w.worse, improved: w.better, flat: "no change", none: "no comparison" }[v]; + let title = v === "none" ? word : `${fmtPct(rel, 2)}: ${word}`; + const unknown = real && opts.noise === null; + if (real && opts.noise != null) title += within ? ` · smaller than twice the series' usual commit-to-commit movement (${fmtAbsPct(opts.noise)}), which changes of this size are common within` : ` · ${(Math.abs(rel) / opts.noise).toFixed(1)}× the series' usual commit-to-commit movement (${fmtAbsPct(opts.noise)})`; + else if (unknown) title += " · no noise estimate yet (fewer than 3 measurements): unconfirmed"; + return h("span", { class: `delta num ${v}${within ? " within" : ""}${unknown ? " unknown" : ""}`, title, "aria-label": title }, + glyph && h("span", { class: "glyph", "aria-hidden": "true" }, glyph), fmtPct(rel, opts.digits ?? 1)); +} +function benchHref(s, { node } = {}) { + const q = new URLSearchParams(); + if (s.params) q.set("params", s.params); + if (s.quantity && s.quantity !== "wall-time") q.set("q", s.quantity); + if (node && multiNode()) q.set("node", node); + const qs = q.toString(); + return `#/b/${encodeURIComponent(s.suite)}/${encodeURIComponent(s.case)}${qs ? "?" + qs : ""}`; +} +function caseLabel(s) { return s.params ? `${s.case} ${s.params}` : s.case; } +function cssVar(name) { return getComputedStyle(document.documentElement).getPropertyValue(name).trim(); } +function nodeColor(node) { + if (!state.nodeSlots.has(node)) state.nodeSlots.set(node, state.nodeSlots.size % SERIES_VARS.length); + return cssVar(SERIES_VARS[state.nodeSlots.get(node)]); +} +function table(headers, rows, opts = {}) { + return h("div", { class: "wrap" }, h("table", {}, + h("thead", {}, h("tr", {}, headers.map((x) => h("th", { class: x.startsWith(">") ? "r" : "" }, x.replace(/^>/, ""))))), + h("tbody", {}, rows.length ? rows.map((cells) => h("tr", {}, cells.map((c, i) => h("td", { class: headers[i].startsWith(">") ? "r num" : "" }, c)))) + : h("tr", {}, h("td", { colspan: headers.length, class: "empty" }, opts.empty || "nothing yet"))))); +} +function tile(label, value, hint, small) { + return h("div", { class: "tile" }, h("div", { class: "label" }, label), h("div", { class: `value ${small ? "small" : ""}` }, value), hint && h("div", { class: "hint" }, hint)); +} +function chip(label, value, tone) { + return h("span", { class: `chip ${tone || ""}` }, h("b", { class: "num" }, value), " ", label); +} +function failTag(reason, text = "failed") { return h("span", { class: "tag failed", title: reason || "" }, text); } +function gapTag(ch) { + if (ch.prev_is_parent) return null; + const title = `The baseline ${shortSha(ch.prev_sha)} is not this commit's parent: the commits in between were not measured for this series, so this change may belong to any of them.`; + return h("span", { class: "tag gap", title }, "not parent"); +} +function stateTag(c) { + if (c.state === "finished") return null; + return h("span", { class: `tag ${c.state === "failed" ? "failed" : ""}` }, c.state === "failed" ? "run failed" : c.state); +} +function sharedLegend(nodes) { + return nodes.length > 1 ? h("span", { class: "legend" }, nodes.map((n) => h("span", { class: "legend-key" }, h("span", { class: "key", style: `border-color:${nodeColor(n)}` }), n))) : null; +} + +// ---- benchmark cards (overview and suite pages) ------------------------------------ + +// groups: one card per (suite, case, params, quantity); its nodes are lines on one chart. +function groupSeries(list, extra = () => ({})) { + const groups = new Map(); + for (const s of list) { + const key = `${s.suite}|${s.case}|${s.params}|${s.quantity}`; + if (!groups.has(key)) groups.set(key, { suite: s.suite, case: s.case, params: s.params, quantity: s.quantity, unit: s.unit, direction: s.direction, list: [] }); + groups.get(key).list.push(s); + } + return [...groups.values()].map((g) => ({ ...g, ...extra(g) })); +} + +function benchCard(g, { showSuite = true } = {}) { + const n = g.newest; + const failed = isFailed(g.status); + const plot = h("div", { class: "spark" }); + const sub = [g.quantity !== "wall-time" ? g.quantity : null, showSuite ? shortSuite(g.suite) : null, n && multiNode() ? n.node : null].filter(Boolean).join(" · "); + const points = n?.points ?? 0; + const card = h("a", { class: `mini${failed ? " failed" : ""}`, href: benchHref(g, { node: n?.node }) }, + h("div", { class: "mini-head" }, + h("span", { class: "mono name", title: caseLabel(g) }, caseLabel(g)), + n ? delta(g.change, g.direction, { noise: n.noise, unit: g.unit }) : null), + h("div", { class: "mini-value" }, h("span", { class: "num" }, n ? fmtValue(n.estimate ?? n.latest, g.unit) : "–"), sub && h("span", { class: "note" }, sub)), + plot, + h("div", { class: "mini-foot" }, + failed ? failTag(g.reason, "failed last run") + : g.status === "skipped" ? h("span", { class: "tag", title: g.reason || "" }, "skipped") + : h("span", {}, points === 1 ? "1 measurement" : `${points} measurements`), + h("span", {}, n?.noise != null ? `noise ${fmtAbsPct(n.noise)}` : points ? "noise: needs 3+" : ""))); + return [card, plot]; +} + +function drawSpark(el, seriesList, axis) { + const index = new Map(axis.map((c, i) => [c.commit_order, i])); + const xs = axis.map((_, i) => i); + const ys = seriesList.map((s) => { const col = new Array(xs.length).fill(null); s.orders.forEach((o, k) => { if (s.estimates[k] != null && index.has(o)) col[index.get(o)] = s.estimates[k]; }); return col; }); + const vals = ys.flat().filter((v) => v != null); + if (!vals.length) { el.append(h("div", { class: "spark-empty" }, "no successful measurement")); return; } + if (typeof uPlot === "undefined") { el.append(h("div", { class: "spark-empty" }, "chart unavailable offline")); return; } + // At least ±3 % of the middle stays visible, so a 0.5 % wiggle looks flat + // instead of filling the card the way a 50 % jump does. + const lo = Math.min(...vals), hi = Math.max(...vals), mid = (lo + hi) / 2; + const half = Math.max((hi - lo) * 0.6, Math.abs(mid) * 0.03, 1e-12); + const width = Math.max(120, el.clientWidth || 240); + const chart = new uPlot({ + width, height: 52, legend: { show: false }, cursor: { show: false }, + scales: { x: { time: false, range: [-0.5, xs.length - 0.5] }, y: { range: [mid - half, mid + half] } }, + axes: [{ show: false }, { show: false }], + series: [{}, ...seriesList.map((s) => ({ stroke: nodeColor(s.node), width: 1.5, spanGaps: true, points: { show: xs.length <= 12, size: 5, stroke: nodeColor(s.node), fill: cssVar("--surface"), width: 1.5 } }))], + }, [xs, ...ys], el); + state.charts.push(chart); +} + +function drawCards(grid, cards, axisFor) { + requestAnimationFrame(() => { for (const [g, plot] of cards) drawSpark(plot, g.list, axisFor(g)); }); +} + +// ---- overview ----------------------------------------------------------------------- + +async function renderLanding() { + const L = state.landing; + if (!L.stats.results) return renderEmpty(); + if (!state.sparks) state.sparks = await getJSON("views/sparks.json"); + const S = state.sparks; + // sparks.json holds successful points only; a series that never succeeded + // still gets a card (an empty chart and its failure) from series.json. + const sparked = new Set(S.series.map((s) => `${s.suite}|${s.case}|${s.params}|${s.quantity}|${s.node}`)); + const never = state.series.filter((s) => !s.points && !sparked.has(`${s.suite}|${s.case}|${s.params}|${s.quantity}|${s.node}`)) + .map((s) => ({ ...s, orders: [], estimates: [], latest: null })); + const all = groupSeries([...S.series, ...never], (g) => { + const newest = g.list.filter((s) => s.latest != null).sort((a, b) => b.orders[b.orders.length - 1] - a.orders[a.orders.length - 1])[0]; + // Status, value and link all describe one node: the newest one's, or the failing one when none succeeded. + const failing = g.list.find((s) => isFailed(s.status) && (!newest || s.node === newest.node)); + const skipped = !failing && g.list.find((s) => s.status === "skipped" && (!newest || s.node === newest.node)); + return { newest, change: newest?.vs_prev ?? null, noise: newest?.noise ?? null, v: newest ? verdict(newest.vs_prev, g.direction) : "none", + status: failing ? failing.status : skipped ? "skipped" : "success", reason: (failing || skipped)?.reason }; + }); + const suites = [...new Set(all.map((g) => g.suite))].sort(); + const nodes = [...new Set(S.series.map((s) => s.node))].sort(); + const q = new URLSearchParams((location.hash.split("?")[1]) || ""); + const ctl = { suite: q.get("suite") || "all", sort: q.get("sort") || "change", text: q.get("f") || "", n: PAGE }; + const setHash = () => { const p = new URLSearchParams(); if (ctl.suite !== "all") p.set("suite", ctl.suite); if (ctl.sort !== "change") p.set("sort", ctl.sort); if (ctl.text) p.set("f", ctl.text); history.replaceState(null, "", "#/" + (p.toString() ? "?" + p : "")); }; + // Missing comparisons sort last; changes within noise rank below real ones of the same size. + const size = (g) => (g.change == null ? -1 : Math.abs(g.change) * (withinNoise(g.change, g.noise) ? 0.5 : 1)); + const sorters = { + change: (a, b) => size(b) - size(a), + regressed: (a, b) => (b.v === "regressed") - (a.v === "regressed") || size(b) - size(a), + improved: (a, b) => (b.v === "improved") - (a.v === "improved") || size(b) - size(a), + failing: (a, b) => isFailed(b.status) - isFailed(a.status) || size(b) - size(a), + noise: (a, b) => (b.noise ?? -1) - (a.noise ?? -1), + name: (a, b) => a.suite.localeCompare(b.suite) || a.case.localeCompare(b.case) || a.params.localeCompare(b.params), + }; + const grid = h("div", { class: "grid" }); + const status = h("span", { class: "note" }); + const more = h("button", { class: "ghost", onclick: () => { ctl.n += PAGE; draw(); } }, `show ${PAGE} more`); + const draw = () => { + const needle = ctl.text.trim().toLowerCase(); + const rows = all.filter((g) => (ctl.suite === "all" || g.suite === ctl.suite) && (!needle || `${g.suite} ${g.case} ${g.params}`.toLowerCase().includes(needle))).sort(sorters[ctl.sort] || sorters.change); + destroyCharts(); + grid.replaceChildren(); + const shown = rows.slice(0, ctl.n), cards = []; + for (const g of shown) { const [card, plot] = benchCard(g); grid.append(card); cards.push([g, plot]); } + if (!shown.length) grid.append(h("p", { class: "empty" }, "no benchmark matches")); + status.textContent = `${shown.length} of ${rows.length} benchmarks`; + more.hidden = shown.length >= rows.length; + drawCards(grid, cards, () => S.axis); + setHash(); + }; + const select = (label, key, values, fmt = (x) => x) => h("label", {}, label, h("select", { onchange: (e) => { ctl[key] = e.target.value; ctl.n = PAGE; draw(); } }, values.map((v) => h("option", { value: v, selected: v === ctl[key] }, fmt(v))))); + const filter = h("input", { type: "search", placeholder: "filter by name", value: ctl.text, "aria-label": "filter benchmarks", oninput: (e) => { ctl.text = e.target.value; ctl.n = PAGE; draw(); } }); + const st = L.stats; + const main = h("main", {}, + problemsBanner(), + ...L.latest_runs.map((run) => latestRun(run, L)), + h("p", { class: "inventory note" }, + `${st.commits} commits measured · ${st.series} series · ${st.results.toLocaleString()} results · `, + st.nodes.length === 1 ? `node ${st.nodes[0]}` : `nodes ${st.nodes.join(", ")}`, " · ", h("a", { href: "#/about" }, "how to read this page")), + h("h2", { id: "all" }, "All benchmarks"), + h("div", { class: "filters" }, + select("suite", "suite", ["all", ...suites], (v) => (v === "all" ? "all suites" : v)), + select("sort", "sort", Object.keys(sorters), (v) => ({ change: "largest change", regressed: "regressed first", improved: "improved first", failing: "failing first", noise: "noisiest", name: "name" })[v]), + filter, status, sharedLegend(nodes)), + h("p", { class: "note" }, `Each chart shows the last ${S.axis.length} measured commits. The change compares the newest measurement with the one before it; muted changes are within twice the series' noise.`), + grid, + h("div", { class: "more" }, more), + h("h2", {}, "Recent commits"), + commitsTable(L.recent_commits, L.notable), + h("div", { class: "cols" }, + h("section", { class: "card" }, h("h3", {}, "Noisiest series"), + h("p", { class: "note" }, "Median change between consecutive measurements, over the last 21. A change on these needs more evidence."), + table(["benchmark", "node", ">noise", ">points"], L.noisiest.map((s) => [h("a", { href: benchHref(s, { node: s.node }) }, caseLabel(s)), s.node, fmtAbsPct(s.noise), s.points]), { empty: "needs at least three measurements per series" })), + h("section", { class: "card" }, h("h3", {}, "Suite wall time on the latest commit"), + table(["suite", "node", ">cases", ">failed", ">time"], L.suite_time.map((s) => [h("a", { href: `#/s/${encodeURIComponent(s.suite)}` }, s.suite), s.node, s.cases, s.failed ? h("span", { class: "bad" }, s.failed) : "0", s.seconds < 120 ? `${s.seconds} s` : `${Math.round(s.seconds / 60)} min`])))), + footer()); + draw(); + return main; +} + +// The first thing on the page: what the newest run on a node found. +function latestRun(run, L) { + const pct = `${Math.round(L.notable * 100)}%`; + const recent = L.last_executed?.[run.node]; + const other = recent && recent.sha !== run.sha ? recent : null; + const mine = (list) => list.filter((m) => m.node === run.node && m.sha === run.sha).slice(0, 5); + const regressed = mine(L.movers.regressed), improved = mine(L.movers.improved); + const failures = L.failures.filter((f) => f.node === run.node); + const moverList = (title, list, tone) => h("div", { class: "movers" }, + h("h3", {}, title), + list.length ? h("ol", {}, list.map((m) => h("li", {}, + h("a", { href: benchHref(m, { node: m.node }), class: "mono name", title: `${caseLabel(m)} (${m.suite})` }, caseLabel(m)), + h("span", { class: "num from-to note" }, `${fmtValue(m.prev_estimate, m.unit)} → ${fmtValue(m.estimate, m.unit)}`), + h("span", { class: "delta-cell" }, m.prev_is_parent ? null : gapTag(m), delta(m.rel_change, m.direction, { noise: m.noise, unit: m.unit }))))) + : h("p", { class: "note" }, !run.compared ? "no earlier measurement of these benchmarks to compare with" + : tone === "bad" ? `nothing got worse by more than 0.5% (${run.compared} compared)` : `nothing got better by more than 0.5% (${run.compared} compared)`)); + const health = run.state !== "finished" ? "bad" : run.failed_results || run.suites_failed ? "warn" : "good"; + const upUrl = upstreamCommit(run.sha); + const logUrl = runLog(run.sha, run.node, run.has_log); + const metaParts = joinParts([ + run.committer_date ? `committed ${fmtDate(run.committer_date)}` : null, + upUrl ? h("a", { href: upUrl, target: "_blank", rel: "noopener" }, "upstream commit") : null, + logUrl ? h("a", { href: logUrl, target: "_blank", rel: "noopener" }, "run log") : null, + ]); + return h("section", { class: `hero ${health}` }, + h("div", { class: "hero-head" }, + h("div", {}, + h("div", { class: "eyebrow" }, `Newest commit measured on ${run.node} · run ${ago(run.finished_at || run.started_at) || fmtDate(run.committer_date)}`), + h("h2", {}, h("a", { href: `#/c/${encodeURIComponent(run.sha)}`, class: "mono" }, run.short), run.subject ? ` ${run.subject}` : ""), + metaParts.length ? h("div", { class: "note" }, ...metaParts) : null), + h("a", { class: "button", href: `#/c/${encodeURIComponent(run.sha)}` }, "All changes in this commit →")), + other ? h("div", { class: `banner ${other.state === "finished" && !other.failed_results ? "" : "bad"}` }, `Most recent run on ${run.node}: `, + h("a", { href: `#/c/${encodeURIComponent(other.sha)}`, class: "mono" }, other.short), ` (an older commit, ${other.reason || "requested"}) finished ${ago(other.finished_at || other.started_at)}: `, + other.state === "finished" ? `${other.results} results, ${other.failed_results} failed.` : h("b", {}, other.state)) : null, + h("div", { class: "chips" }, + chip("results", run.results.toLocaleString()), + chip("failed", run.failed_results, run.failed_results ? "bad" : "good"), + chip(`regressed > ${pct}`, run.regressed, run.regressed ? "bad" : ""), + chip(`improved > ${pct}`, run.improved, run.improved ? "good" : ""), + chip("compared with an earlier run", run.compared), + run.state !== "finished" ? chip("", run.state, "bad") : null), + h("div", { class: "cols tight" }, moverList("Largest regressions", regressed, "bad"), moverList("Largest improvements", improved, "good")), + failures.length ? h("details", { class: "failures", open: failures.length <= 6 }, + h("summary", {}, h("span", { class: "bad" }, `${failures.length} benchmark${failures.length > 1 ? "s" : ""} failed`), " on this run"), + h("ul", {}, failures.map((f) => h("li", {}, h("a", { href: benchHref(f, { node: f.node }), class: "mono" }, caseLabel(f)), " ", h("span", { class: "note" }, shortSuite(f.suite)), " ", + h("code", { class: "reason" }, f.reason || f.status), f.last_success_sha ? null : h("span", { class: "note" }, " · never succeeded"))))) + : null); +} + +function commitsTable(list, notable) { + const pct = `${Math.round(notable * 100)}%`; + return table(["commit", "subject", "date", "node", ">results", ">failed", `>regressed > ${pct}`, `>improved > ${pct}`], + list.map((c) => [ + h("a", { href: `#/c/${encodeURIComponent(c.sha)}`, class: "mono" }, c.short), + h("span", {}, c.subject.length > 90 ? c.subject.slice(0, 88) + "…" : c.subject, " ", stateTag(c)), + fmtDate(c.committer_date), c.node, c.results, + c.failed_results ? h("span", { class: "bad" }, c.failed_results) : "0", + c.compared ? (c.regressed ? h("span", { class: "delta regressed" }, c.regressed) : "0") : "–", + c.compared ? (c.improved ? h("span", { class: "delta improved" }, c.improved) : "0") : "–"]), + { empty: "no runs filed yet" }); +} + +function problemsBanner() { + const n = state.manifest.counts.ingest_problems || 0; + return n ? h("div", { class: "banner bad" }, `${n} result${n > 1 ? "s or run directories were" : " or run directory was"} rejected by the ingest and ${n > 1 ? "are" : "is"} missing from this page.`) : null; +} + +// No results yet: say what state the project is in and what to do next. +function renderEmpty() { + const L = state.landing, repo = state.project.repository, isRuns = state.manifest?.source === "runs"; + return h("main", {}, + problemsBanner(), + h("section", { class: "hero" }, + h("div", { class: "eyebrow" }, "No results yet"), + h("h2", {}, L.recent_commits.length ? "Runs are filed, but none produced a result" : "Waiting for the first run"), + h("p", {}, L.recent_commits.length + ? "The runs below have no successful measurements. Open a run to inspect its status and log." + : isRuns + ? "This page fills in when benchmark runs are filed under runs/. Run a benchmark and rebuild the site with bx view." + : "This store has no benchmark measurements yet. Measure a target with bx bench, then run bx view:"), + L.recent_commits.length ? null : h("pre", {}, isRuns + ? "bx view --runs runs --out site" + : "bx bench path/to/benchmarks.py\nbx view"), + repo ? h("p", {}, h("a", { href: repo }, "Project repository")) : null), + L.recent_commits.length ? [h("h2", {}, "Runs"), commitsTable(L.recent_commits, L.notable)] : null, + footer()); +} + +// ---- suites ------------------------------------------------------------------------- + +function renderSuites() { + const by = new Map(); + for (const s of state.series) { + const e = by.get(s.suite) || { suite: s.suite, series: 0, cases: new Set(), nodes: new Set(), regressed: 0, improved: 0, failing: 0 }; + e.series++; e.cases.add(`${s.case}|${s.params}`); e.nodes.add(s.node); + if (isFailed(s.status)) e.failing++; + else if (s.vs_prev != null && Math.abs(s.vs_prev) > state.landing.notable) { + const v = verdict(s.vs_prev, s.direction); + if (v === "regressed") e.regressed++; else if (v === "improved") e.improved++; + } + by.set(s.suite, e); + } + const rows = [...by.values()].sort((a, b) => b.failing - a.failing || b.regressed - a.regressed || a.suite.localeCompare(b.suite)); + const pct = `${Math.round(state.landing.notable * 100)}%`; + return h("main", {}, + h("nav", { class: "crumbs" }, h("a", { href: "#/" }, "overview"), " / suites"), + h("h2", {}, "Suites"), + h("p", { class: "lead" }, `${rows.length} benchmark suite${rows.length === 1 ? "" : "s"}. Regressed and improved count series whose latest change exceeds ${pct}; failing counts series whose latest attempt did not succeed.`), + table(["suite", ">benchmarks", ">series", "nodes", ">failing", ">regressed", ">improved"], rows.map((e) => [ + h("a", { href: `#/s/${encodeURIComponent(e.suite)}` }, e.suite), e.cases.size, e.series, [...e.nodes].sort().join(", "), + e.failing ? h("span", { class: "bad" }, e.failing) : "0", + e.regressed ? h("span", { class: "delta regressed" }, e.regressed) : "0", e.improved ? h("span", { class: "delta improved" }, e.improved) : "0"])), + footer()); +} + +async function loadHistory(suite) { + let hist = state.histories.get(suite); + if (!hist) { hist = await getJSON(`views/history/${encodeURIComponent(safeFile(suite))}.json`); state.histories.set(suite, hist); } + return hist; +} + +async function renderSuite(suite) { + const rows = state.series.filter((s) => s.suite === suite); + const nodes = [...new Set(rows.map((r) => r.node))].sort(); + const hist = await loadHistory(suite); + const latestBy = new Map(rows.map((r) => [`${r.case}|${r.params}|${r.quantity}|${r.node}`, r])); + const groups = groupSeries(Object.values(hist.series).map((s) => ({ ...s, suite })), (g) => { + const mine = g.list.map((s) => latestBy.get(`${s.case}|${s.params}|${s.quantity}|${s.node}`)).filter(Boolean); + const newest = mine.filter((r) => r.estimate != null).sort((a, b) => b.commit_order - a.commit_order)[0]; + const failing = mine.find((r) => isFailed(r.status) && (!newest || r.node === newest.node)); + const skipped = !failing && mine.find((r) => r.status === "skipped" && (!newest || r.node === newest.node)); + return { newest, change: newest?.vs_prev ?? null, status: failing ? failing.status : skipped ? "skipped" : "success", reason: (failing || skipped)?.reason }; + }).sort((a, b) => a.case.localeCompare(b.case) || a.params.localeCompare(b.params)); + const grid = h("div", { class: "grid" }); + const status = h("span", { class: "note" }); + const drawSuite = (needle = "") => { + destroyCharts(); + grid.replaceChildren(); + const shown = groups.filter((g) => !needle || caseLabel(g).toLowerCase().includes(needle)); + const cards = shown.map((g) => { const [card, plot] = benchCard(g, { showSuite: false }); grid.append(card); return [g, plot]; }); + status.textContent = `${shown.length} of ${groups.length} benchmarks`; + drawCards(grid, cards, () => hist.commits); + }; + const filter = h("input", { type: "search", placeholder: "filter by name", "aria-label": "filter benchmarks", oninput: (e) => drawSuite(e.target.value.trim().toLowerCase()) }); + const failing = groups.filter((g) => isFailed(g.status)).length; + const main = h("main", {}, + h("nav", { class: "crumbs" }, h("a", { href: "#/" }, "overview"), " / ", h("a", { href: "#/suites" }, "suites"), " / ", suite), + h("h2", {}, suite), + h("p", { class: "lead" }, `${groups.length} benchmarks over ${hist.commits.length} measured commits`, failing ? [", ", h("span", { class: "bad" }, `${failing} failing`)] : null, `. The change compares each benchmark's newest measurement with the one before it.`), + h("div", { class: "filters" }, filter, status, sharedLegend(nodes)), + grid, + h("details", { class: "table-view" }, h("summary", {}, "Table view"), + table(["benchmark", "quantity", "node", "status", ">latest", ">vs previous", ">vs 10 back", ">noise", ">points"], + rows.map((s) => [h("a", { href: benchHref(s, { node: s.node }) }, caseLabel(s)), s.quantity, s.node, + isFailed(s.status) ? failTag(s.reason, s.status) : s.status === "skipped" || s.status === "partial" ? h("span", { class: "tag", title: s.reason || "" }, s.status) : "ok", + fmtValue(s.estimate, s.unit), delta(s.vs_prev, s.direction, { noise: s.noise, unit: s.unit }), delta(s.vs_10, s.direction, { unit: s.unit }), fmtAbsPct(s.noise), s.points]))), + footer()); + drawSuite(); + return main; +} + +// ---- one benchmark ---------------------------------------------------------------- + +async function renderBench(suite, kase, query) { + const params = query.get("params") || ""; + const quantity = query.get("q") || "wall-time"; + const nodeFilter = query.get("node") || "all"; + const hist = await loadHistory(suite); + const all = Object.values(hist.series).filter((s) => s.case === kase); + const variants = [...new Set(all.map((s) => s.params))]; + const quantities = [...new Set(all.map((s) => s.quantity))]; + const nodes = [...new Set(all.map((s) => s.node))].sort(); + const shown = all.filter((s) => s.params === params && s.quantity === quantity && (nodeFilter === "all" || s.node === nodeFilter)); + const latestRows = state.series.filter((s) => s.suite === suite && s.case === kase && s.params === params && s.quantity === quantity && (nodeFilter === "all" || s.node === nodeFilter)); + const kpi = latestRows.filter((r) => r.estimate != null).sort((a, b) => b.commit_order - a.commit_order)[0]; + const failing = latestRows.filter((r) => isFailed(r.status)); + const unit = shown[0]?.unit || kpi?.unit || ""; + const direction = shown[0]?.direction || kpi?.direction || ""; + const opts = { unit }; + + const setParam = (k, v) => { const q = new URLSearchParams(query); if (v == null || v === "" || v === "all" || (k === "q" && v === "wall-time")) q.delete(k); else q.set(k, v); location.hash = `#/b/${encodeURIComponent(suite)}/${encodeURIComponent(kase)}${q.toString() ? "?" + q : ""}`; }; + const select = (label, key, values, current, fmt = (x) => x) => h("label", {}, label, h("select", { onchange: (e) => setParam(key, e.target.value) }, values.map((v) => h("option", { value: v, selected: v === current }, fmt(v))))); + const few = kpi && kpi.points < 3; + const lastNs = shown.flatMap((x) => x.ns.filter((n, k) => x.estimates[k] != null)); + const reps = lastNs.length ? lastNs[lastNs.length - 1] : null; + + const main = h("main", {}, + h("nav", { class: "crumbs" }, h("a", { href: "#/" }, "overview"), " / ", h("a", { href: `#/s/${encodeURIComponent(suite)}` }, suite)), + h("h2", { class: "mono title" }, kase, params ? ` ${params}` : ""), + h("p", { class: "lead" }, `${quantity}, ${direction ? direction.replace(/-/g, " ") : "direction unknown"}`, kpi ? ` · ${kpi.node}` : ""), + h("div", { class: "filters" }, + variants.length > 1 ? select("variant", "params", variants, params, (v) => v || "(default)") : null, + quantities.length > 1 ? select("quantity", "q", quantities, quantity) : null, + nodes.length > 1 ? select("node", "node", ["all", ...nodes], nodeFilter) : null), + failing.map((f) => h("div", { class: "banner bad" }, `The latest attempt on ${f.node} (commit `, h("a", { href: `#/c/${encodeURIComponent(f.attempt_sha)}`, class: "mono" }, shortSha(f.attempt_sha)), ") failed: ", + h("code", {}, f.reason || f.status), f.points ? ". The values below are from the last successful measurement." : ". It has never succeeded.")), + kpi ? h("div", { class: "tiles" }, + tile(`latest · ${kpi.node}`, fmtValue(kpi.estimate, unit), h("span", {}, "commit ", h("a", { href: `#/c/${encodeURIComponent(kpi.sha)}`, class: "mono" }, shortSha(kpi.sha)))), + tile("vs previous measurement", delta(kpi.vs_prev, direction, { ...opts, noise: kpi.noise })), + tile("vs 10 measurements back", delta(kpi.vs_10, direction, opts), kpi.points <= 10 ? `needs 11+, has ${kpi.points}` : null), + tile("vs 100 measurements back", delta(kpi.vs_100, direction, opts), kpi.points <= 100 ? `needs 101+, has ${kpi.points}` : null), + tile("noise", few ? "–" : fmtAbsPct(kpi.noise), few ? "needs 3+ measurements" : kpi.noise_rank != null ? `quieter than ${Math.round((1 - kpi.noise_rank) * 100)}% of the suite` : null), + tile("measured commits", kpi.points, reps == null ? null : reps === 0 ? "producer summary" : `${reps} repetition${reps === 1 ? "" : "s"} each, median shown`)) : failing.length ? null : h("p", { class: "empty" }, "no successful measurement yet"), + few ? h("p", { class: "note" }, `With ${kpi.points} measurement${kpi.points > 1 ? "s" : ""} there is no noise estimate yet, so a change here cannot be told apart from run-to-run variation.`) : null, + h("h3", {}, "History"), + h("p", { class: "note" }, "One point per measured commit, in commit order. Hover for details, click a point to open its commit, drag to zoom, double-click to reset. Failed attempts are marked under the axis.")); + const chartWrap = h("div", { class: "chart-wrap" }); + main.append(chartWrap); + const moves = biggestMoves(shown, hist.commits); + main.append(h("h3", {}, "Biggest moves in this series"), + table(["commit", "subject", "node", "baseline", ">from", ">to", ">change"], moves.slice(0, 12).map((m) => [ + h("a", { href: `#/c/${encodeURIComponent(m.sha)}`, class: "mono" }, m.short), m.subject.length > 70 ? m.subject.slice(0, 68) + "…" : m.subject, m.node, + h("span", {}, h("a", { href: `#/c/${encodeURIComponent(m.baseSha)}`, class: "mono" }, shortSha(m.baseSha, 7)), + m.skipped > 0 ? h("span", { class: "tag gap", title: `${m.skipped} measured commit${m.skipped > 1 ? "s" : ""} in between had no successful result for this series, so this change spans ${m.skipped > 1 ? "them" : "it"}` }, `spans ${m.skipped + 1}`) : null), + fmtValue(m.from, unit), fmtValue(m.to, unit), delta(m.rel, direction, { ...opts, noise: kpi?.noise })]), { empty: "needs two successful measurements" }), + footer()); + requestAnimationFrame(() => drawHistory(chartWrap, shown, hist.commits, unit, kase)); + return main; +} + +function biggestMoves(seriesList, axis) { + const bySha = new Map(axis.map((c) => [c.commit_order, c])); + const position = new Map(axis.map((c, i) => [c.commit_order, i])); + const out = []; + for (const s of seriesList) { + let prev = null, prevOrder = null; + for (let i = 0; i < s.orders.length; i++) { + const b = s.estimates[i]; + if (b == null) continue; + if (prev != null && prev > 0) { + const c = bySha.get(s.orders[i]); + const base = bySha.get(prevOrder); + // skipped: the suite's measured commits between the baseline and this one + out.push({ sha: c?.sha || "", short: c?.short || "", subject: c?.subject || "", node: s.node, from: prev, to: b, rel: b / prev - 1, + baseSha: base?.sha || "", skipped: position.get(s.orders[i]) - position.get(prevOrder) - 1 }); + } + prev = b; prevOrder = s.orders[i]; + } + } + return out.sort((x, y) => Math.abs(y.rel) - Math.abs(x.rel)); +} + +function drawHistory(wrap, seriesList, axis, unit, title) { + wrap.replaceChildren(); + if (!seriesList.length || !axis.length) { wrap.append(h("div", { class: "empty" }, "no points to draw")); return; } + const index = new Map(axis.map((c, i) => [c.commit_order, i])); + const xs = axis.map((_, i) => i); + const allValues = seriesList.flatMap((s) => s.estimates); + const scale = pickScale(allValues, unit); + const ys = seriesList.map((s) => { const col = new Array(xs.length).fill(null); s.orders.forEach((o, k) => { if (s.estimates[k] != null) col[index.get(o)] = s.estimates[k] / scale[1]; }); return col; }); + const fails = seriesList.map((s) => { const col = new Array(xs.length).fill(null); s.orders.forEach((o, k) => { if (isFailed(s.statuses[k])) col[index.get(o)] = s.reasons?.[k] || s.statuses[k]; }); return col; }); + const spread = seriesList.map((s) => { const col = new Array(xs.length).fill(null); s.orders.forEach((o, k) => { if (s.estimates[k] != null) col[index.get(o)] = { n: s.ns?.[k], min: s.mins?.[k], max: s.maxs?.[k] }; }); return col; }); + const nFailed = fails.flat().filter(Boolean).length; + const totalPoints = allValues.filter((v) => v != null).length; + const tableView = (open = false) => h("details", { class: "table-view", open }, h("summary", {}, "Table view"), + table(["commit", "date", ...seriesList.map((s) => `>${s.node} (${scale[0]})`)], axis.map((c, i) => [h("a", { href: `#/c/${encodeURIComponent(c.sha)}`, class: "mono" }, c.short), fmtDate(c.date), + ...ys.map((col, k) => (col[i] != null ? fmtNum(col[i]) : fails[k][i] ? failTag(fails[k][i]) : "–"))]))); + if (!totalPoints) { + wrap.append(h("div", { class: "empty" }, `No successful measurement to draw: ${nFailed} failed attempt${nFailed === 1 ? "" : "s"} over ${axis.length} commits. The table below has the reason for each.`), + h("details", { class: "table-view", open: true }, h("summary", {}, "Table view"), + table(["commit", "date", ...seriesList.map((s) => `${s.node}`)], axis.map((c, i) => [h("a", { href: `#/c/${encodeURIComponent(c.sha)}`, class: "mono" }, c.short), fmtDate(c.date), ...fails.map((col) => (col[i] ? failTag(col[i], col[i]) : "–"))])))); + return; + } + if (typeof uPlot === "undefined") { + wrap.append(h("div", { class: "empty" }, "Chart library (uPlot) could not be loaded offline. Use the table below."), tableView(true)); + return; + } + const tip = h("div", { class: "tip" }); + const titleRow = h("div", { class: "chart-title" }, h("span", {}, `${title} · ${scale[0]}`), h("span", { class: "note" }, `${axis.length} commits`, nFailed ? [" · ", h("span", { class: "bad" }, `${nFailed} failed`)] : null)); + const el = h("div", { class: "chart" }); + wrap.append(titleRow, el, tip); + const size = () => ({ width: Math.max(240, wrap.clientWidth - 16), height: window.innerWidth < 600 ? 220 : 300 }); + const failMarks = (u) => { // a small mark under the plot at every failed attempt + const ctx = u.ctx, y = u.bbox.top + u.bbox.height + 3 * devicePixelRatio; + ctx.save(); ctx.fillStyle = cssVar("--critical"); + fails.forEach((col) => col.forEach((f, i) => { if (!f) return; const x = u.valToPos(i, "x", true); ctx.fillRect(x - 3 * devicePixelRatio, y, 6 * devicePixelRatio, 3 * devicePixelRatio); })); + ctx.restore(); + }; + const opts = { + ...size(), + legend: { show: seriesList.length > 1, live: false }, + cursor: { points: { size: 10 }, drag: { x: true, y: false } }, + scales: { x: { time: false, range: [-0.5, xs.length - 0.5] }, y: { range: (u, min, max) => { const pad = (max - min) * 0.25 || max * 0.1 || 1; return [Math.max(0, min - pad), max + pad]; } } }, + axes: [ + { stroke: cssVar("--muted"), grid: { show: false }, ticks: { stroke: cssVar("--axis"), width: 1 }, font: "12px system-ui, sans-serif", space: 70, gap: 8, + splits: (u, axisIdx, min, max, incr) => { const step = Math.max(1, Math.ceil(incr)); const out = []; for (let i = Math.max(0, Math.ceil(min)); i <= Math.min(xs.length - 1, Math.floor(max)); i += step) out.push(i); return out; }, + values: (u, vals) => vals.map((v) => axis[v] ? (axis[v].short.includes("+dirty") ? axis[v].short.slice(0, 13) : axis[v].short.slice(0, 7)) : "") }, + { stroke: cssVar("--muted"), grid: { stroke: cssVar("--grid"), width: 1 }, ticks: { show: false }, font: "12px system-ui, sans-serif", size: 60, + values: (u, vals) => vals.map((v) => fmtNum(v, 3)) }, + ], + series: [{ label: "commit" }, ...seriesList.map((s) => ({ + label: s.node, stroke: nodeColor(s.node), width: 2, spanGaps: true, + points: { show: totalPoints <= 80, size: 8, stroke: nodeColor(s.node), fill: cssVar("--surface"), width: 2 }, + }))], + hooks: { + draw: [failMarks], + setCursor: [(u) => { + const i = u.cursor.idx; + if (i == null || !axis[i]) { tip.style.display = "none"; return; } + const c = axis[i]; + const rows = seriesList.map((s, k) => { + const v = ys[k][i]; + let prev = null; + for (let j = i - 1; j >= 0; j--) if (ys[k][j] != null) { prev = ys[k][j]; break; } + const rel = v != null && prev > 0 ? v / prev - 1 : null; + return h("div", { class: "row" }, h("span", { class: "key", style: `border-color:${nodeColor(s.node)}` }), + h("span", { class: "v num" }, v != null ? `${fmtNum(v)} ${scale[0]}` : fails[k][i] ? h("span", { class: "bad" }, `failed: ${fails[k][i]}`) : "not measured"), + h("span", { class: "s num" }, rel == null ? "" : fmtPct(rel)), seriesList.length > 1 ? h("span", { class: "s" }, s.node) : null, + spread[k][i] ? h("span", { class: "s spread" }, spread[k][i].n > 1 + ? `median of ${spread[k][i].n}, range ${fmtNum(spread[k][i].min / scale[1])}–${fmtNum(spread[k][i].max / scale[1])}` + : spread[k][i].n === 1 ? "1 repetition: no spread" : "summary only: no spread") : null); + }); + tip.replaceChildren(h("div", { class: "when" }, `${c.short} · ${fmtDate(c.date)}`), ...rows, h("div", { class: "subj" }, c.subject)); + tip.style.display = "block"; + const left = u.cursor.left, bbox = u.over.getBoundingClientRect(), wrapBox = wrap.getBoundingClientRect(); + const x = bbox.left - wrapBox.left + left + 14, maxX = wrap.clientWidth - tip.offsetWidth - 8; + tip.style.left = `${Math.max(8, Math.min(x, maxX))}px`; + tip.style.top = `${bbox.top - wrapBox.top + 10}px`; + }], + }, + }; + const chart = new uPlot(opts, [xs, ...ys], el); + state.charts.push(chart); + el.addEventListener("mouseleave", () => { tip.style.display = "none"; }); + el.addEventListener("click", () => { const i = chart.cursor.idx; if (i != null && axis[i] && chart.select.width === 0) location.hash = `#/c/${encodeURIComponent(axis[i].sha)}`; }); + wrap.append(tableView(false)); + const onResize = () => chart.setSize(size()); + window.addEventListener("resize", onResize); + state.cleanup.push(() => window.removeEventListener("resize", onResize)); +} + +// ---- one commit --------------------------------------------------------------------- + +async function renderCommit(sha) { + let c = state.commits.get(sha); + if (!c) { c = await getJSON(`views/commits/${encodeURIComponent(safeFile(sha))}.json`); state.commits.set(sha, c); } + const filter = h("input", { type: "search", placeholder: "filter benchmarks", "aria-label": "filter benchmarks" }); + const minPct = h("select", {}, [0, 1, 2, 5, 10].map((p) => h("option", { value: p, selected: p === 0 }, p ? `≥ ${p}%` : "all changes"))); + const hideNoise = h("input", { type: "checkbox" }); + const body = h("div", {}); + const draw = () => { + const needle = filter.value.trim().toLowerCase(), min = Number(minPct.value) / 100; + const rows = c.changes.filter((ch) => Math.abs(ch.rel_change) >= min && !(hideNoise.checked && withinNoise(ch.rel_change, ch.noise)) + && (!needle || `${ch.suite} ${ch.case} ${ch.params} ${ch.node}`.toLowerCase().includes(needle))); + body.replaceChildren(table(["benchmark", "suite", "node", ">baseline", ">before", ">after", ">change", ">noise"], rows.map((ch) => [ + h("a", { href: benchHref(ch, { node: ch.node }) }, caseLabel(ch)), shortSuite(ch.suite), ch.node, + h("span", {}, gapTag(ch), " ", h("a", { href: `#/c/${encodeURIComponent(ch.prev_sha)}`, class: "mono" }, shortSha(ch.prev_sha, 7))), + fmtValue(ch.prev_estimate, ch.unit), fmtValue(ch.estimate, ch.unit), + delta(ch.rel_change, ch.direction, { noise: ch.noise, unit: ch.unit }), fmtAbsPct(ch.noise)]), { empty: "no changes match" }), + h("p", { class: "note" }, `${rows.length} of ${c.changes.length} compared series`)); + }; + filter.addEventListener("input", draw); minPct.addEventListener("change", draw); hideNoise.addEventListener("change", draw); + const parentMeasured = c.changes.some((ch) => ch.prev_is_parent); + const upUrl = upstreamCommit(c.sha); + const leadParts = joinParts([ + c.committer_date ? fmtDate(c.committer_date) : null, + upUrl ? h("a", { href: upUrl, target: "_blank", rel: "noopener" }, "upstream commit") : null, + c.parent ? h("span", {}, "parent ", h("a", { href: `#/c/${encodeURIComponent(c.parent)}`, class: "mono" }, shortSha(c.parent))) : null, + ]); + const main = h("main", {}, + h("nav", { class: "crumbs" }, h("a", { href: "#/" }, "overview"), " / commit"), + h("h2", {}, h("span", { class: "mono" }, c.short), c.subject ? ` ${c.subject}` : ""), + leadParts.length ? h("p", { class: "lead" }, ...leadParts) : null, + h("h3", {}, "Runs"), + table(["node", "state", "reason", "started", "finished", ">suites ok", ">failed suites", ">results", ">failed results", ""], c.nodes.map((n) => { + const filesUrl = runFiles(c.sha, n.node); + const logUrl = runLog(c.sha, n.node, n.has_log); + const links = joinParts([ + filesUrl ? h("a", { href: filesUrl, target: "_blank", rel: "noopener" }, "parquet") : null, + logUrl ? h("a", { href: logUrl, target: "_blank", rel: "noopener" }, "log") : null, + ]); + return [ + n.node, n.state === "finished" ? n.state : h("span", { class: "bad" }, n.state), n.reason || "", fmtDate(n.started_at), fmtDate(n.finished_at), + n.suites_ok, n.suites_failed ? h("span", { class: "bad" }, n.suites_failed) : "0", n.results, + n.failed_results ? h("span", { class: "bad" }, n.failed_results) : "0", + links.length ? h("span", {}, ...links) : ""]; + })), + c.failures?.length ? [h("h3", {}, h("span", { class: "bad" }, `Failed results (${c.failures.length})`)), + table(["benchmark", "suite", "node", "quantity", "reason"], c.failures.map((f) => [h("a", { href: benchHref(f, { node: f.node }) }, caseLabel(f)), shortSuite(f.suite), f.node, f.quantity, h("code", {}, f.reason || f.status)]))] : null, + h("h3", {}, "Changes against the previous measured point"), + h("p", { class: "note" }, parentMeasured || !c.changes.length + ? "Each series is compared with its own previous successful measurement, named in the baseline column. \"not parent\" marks a baseline further back than the parent commit." + : "This commit's parent was not measured, so every change here also covers the unmeasured commits in between."), + h("div", { class: "filters" }, h("label", {}, "show", minPct), h("label", {}, hideNoise, "hide changes within noise"), filter), + body, footer()); + draw(); + return main; +} + +// ---- about: how to read the page ----------------------------------------------------- + +function renderAbout() { + const P = state.project, repo = P.repository, pct = `${Math.round(state.landing.notable * 100)}%`; + const upstream = (P.upstream?.url || "").replace(/\.git$/, ""); + const isRuns = state.manifest?.source === "runs"; + return h("main", { class: "prose" }, + h("nav", { class: "crumbs" }, h("a", { href: "#/" }, "overview"), " / about"), + h("h2", {}, "About this page"), + h("p", { class: "lead" }, P.description || "Benchmark results and history, built with benchx."), + h("div", { class: "cols" }, + h("section", { class: "card" }, h("h3", {}, isRuns ? "How a commit gets measured" : "How this report is built"), + isRuns + ? h("ol", {}, + h("li", {}, "Commits from ", upstream ? h("a", { href: upstream }, `${upstream.replace(/^https:\/\/github.com\//, "")} ${P.upstream?.branch || ""}`.trim()) : "the upstream repository", " are built and benchmarked through benchx."), + h("li", {}, "Each run is filed as Parquet under ", h("code", {}, "runs///"), "."), + h("li", {}, h("code", {}, "bx view"), " derives the flat tables and static JSON views this page reads. Nothing is stored in a database or server.")) + : h("ol", {}, + h("li", {}, "Benchmarks are executed with ", h("code", {}, "bx bench"), " or ", h("code", {}, "bx compare"), ", recording results in the benchx schema."), + h("li", {}, h("code", {}, "bx view"), " reads the local store (", h("code", {}, "store.parquet"), ") or JSON result files and builds these static views."), + h("li", {}, "The same static site can be viewed locally or published to static hosting without a backend."))), + h("section", { class: "card" }, h("div", { class: "about-brand" }, wordmark()), h("h3", {}, "Built with benchx"), + h("p", {}, "Every number shown on this page is derived directly from stored benchx measurement results."), + h("p", {}, h("a", { class: "button", href: "https://github.com/benchx-dev/benchx" }, "benchx on GitHub →"), + repo ? [" ", h("a", { href: repo }, "project repository")] : null))), + h("h3", {}, "Reading the numbers"), + h("dl", {}, + h("dt", {}, "Change"), h("dd", {}, "A benchmark's newest successful measurement divided by its previous successful one, minus one. ", delta(0.12, "lower-is-better", { unit: "s", noise: 0.01 }), " is worse, ", delta(-0.12, "lower-is-better", { unit: "s", noise: 0.01 }), " is better; for higher-is-better quantities such as throughput the colors flip. The arrow is the direction of the number; the color is good or bad. Under 0.5% is shown as flat (", delta(0.001, "lower-is-better"), ")."), + h("dt", {}, "Noise"), h("dd", {}, "The series' usual commit-to-commit movement: the median change between consecutive measurements over its last 21. It is historical variability across commits, not repeatability at one commit, and it is the series' current value, also on older commit pages. It needs three measurements. A change smaller than twice the noise is drawn muted (", delta(0.03, "lower-is-better", { unit: "s", noise: 0.02 }), "), because changes that size are common for this series. A change with no noise estimate yet is underlined (", delta(0.12, "lower-is-better", { unit: "s", noise: null }), "): unconfirmed."), + h("dt", {}, "Measured commits and repetitions"), h("dd", {}, "Each point is the median of the repetitions benchx ran on that commit. The history chart's tooltip shows how many there were and their range. With one repetition there is no spread to judge a point by."), + h("dt", {}, "Not a verdict"), h("dd", {}, "None of this is a statistical test yet. benchx's comparator history mode will compare each commit with its ancestors using the measured repetitions; until then, treat a change as a lead to check, and re-measure it before acting on it."), + h("dt", {}, "Baseline and \"not parent\""), h("dd", {}, "A change is against the previous measurement of the same series, which is the parent commit only if the parent was measured. Otherwise the change spans every unmeasured commit in between, and the commit page marks it ", h("span", { class: "tag gap" }, "not parent"), "."), + h("dt", {}, "Counts"), h("dd", {}, `"Regressed" and "improved" in the summaries count changes larger than ${pct} in either direction, whatever the noise.`), + h("dt", {}, "Failures"), h("dd", {}, "A benchmark whose latest attempt failed is marked on its card and listed under the latest run, with the reason benchx recorded. Its chart keeps the last successful values, and failed attempts are marked under the history chart's axis.")), + footer()); +} + +function footer() { + const m = state.manifest, repo = state.project.repository; + const hasTables = m.tables && Object.keys(m.tables).length > 0; + const t = (name) => h("a", { href: `${DATA}${m.tables[name]}`, download: true }, name); + return h("footer", {}, + h("span", {}, `built ${fmtDate(m.built_at)} · ${m.counts.points.toLocaleString()} results`, + hasTables ? [" · data: ", ...joinParts(["points", "latest", "changes", "commits"].filter((k) => m.tables[k]).map(t), ", "), " (Parquet)"] : null), + h("span", { class: "made" }, "measured with ", h("a", { href: "https://github.com/benchx-dev/benchx", title: "benchx" }, wordmark(true)), repo ? [" · ", h("a", { href: repo }, "source")] : null)); +} + +// ---- shell: header, search, theme, router ------------------------------------------- + +function header() { + const input = h("input", { type: "search", placeholder: "search a benchmark… ( / )", "aria-label": "search a benchmark", autocomplete: "off" }); + const list = h("ul", { role: "listbox", hidden: true }); + let items = [], active = -1; + const show = () => { + const needle = input.value.trim().toLowerCase(); + list.replaceChildren(); active = -1; + if (!needle) { list.hidden = true; return; } + const seen = new Set(); + items = []; + for (const s of state.series) { + const key = `${s.suite}|${s.case}|${s.params}|${s.quantity}`; + if (seen.has(key)) continue; + if (`${s.case} ${s.params} ${s.suite}`.toLowerCase().includes(needle)) { seen.add(key); items.push(s); } + if (items.length >= 14) break; + } + list.hidden = items.length === 0; + items.forEach((s) => list.append(h("li", { role: "option", onclick: () => go(s) }, h("span", { class: "mono" }, caseLabel(s)), h("span", { class: "where" }, s.suite)))); + }; + const go = (s) => { input.value = ""; list.hidden = true; location.hash = benchHref(s); }; + input.addEventListener("input", show); + input.addEventListener("keydown", (e) => { + if (list.hidden) return; + if (e.key === "ArrowDown") { active = Math.min(active + 1, items.length - 1); e.preventDefault(); } + else if (e.key === "ArrowUp") { active = Math.max(active - 1, 0); e.preventDefault(); } + else if (e.key === "Enter") { if (items.length) go(items[Math.max(active, 0)]); return; } + else if (e.key === "Escape") { list.hidden = true; return; } + [...list.children].forEach((li, i) => li.setAttribute("aria-selected", i === active)); + }); + document.addEventListener("click", (e) => { if (!list.contains(e.target) && e.target !== input) list.hidden = true; }); + document.addEventListener("keydown", (e) => { if (e.key === "/" && !/INPUT|SELECT|TEXTAREA/.test(document.activeElement?.tagName)) { e.preventDefault(); input.focus(); } }); + const label = () => `theme: ${document.documentElement.dataset.theme || "auto"}`; + const theme = h("button", { class: "ghost", title: "cycle theme: auto / dark / light", onclick: (e) => { const cur = document.documentElement.dataset.theme || "auto"; applyTheme(cur === "auto" ? "dark" : cur === "dark" ? "light" : "auto"); e.currentTarget.textContent = label(); route(); } }, label()); + const nav = (href, text) => h("a", { href, "data-nav": href }, text); + return h("header", { class: "top" }, + h("h1", {}, h("a", { href: "#/", class: "brand" }, h("span", { html: MARK }), state.project.title || "Benchmarks")), + h("nav", {}, nav("#/", "overview"), nav("#/suites", "suites"), nav("#/about", "about")), + h("div", { class: "search" }, input, list), theme); +} + +function applyTheme(mode) { + if (mode === "auto") delete document.documentElement.dataset.theme; else document.documentElement.dataset.theme = mode; + try { if (mode === "auto") localStorage.removeItem("theme"); else localStorage.setItem("theme", mode); } catch (e) { /* storage unavailable */ } +} + +function destroyCharts() { + for (const c of state.charts) c.destroy(); + state.charts = []; +} + +async function route() { + const root = document.getElementById("app"); + const hash = location.hash || "#/"; + const [path, qs] = hash.slice(1).split("?"); + const query = new URLSearchParams(qs || ""); + const parts = path.split("/").filter(Boolean).map(decodeURIComponent); + destroyCharts(); + for (const f of state.cleanup) f(); + state.cleanup = []; + const section = parts[0] === "suites" || parts[0] === "s" ? "#/suites" : parts[0] === "about" ? "#/about" : parts[0] ? null : "#/"; + document.querySelectorAll("[data-nav]").forEach((a) => a.toggleAttribute("aria-current", a.dataset.nav === section)); + let view; + try { + if (parts[0] === "suites") view = renderSuites(); + else if (parts[0] === "about") view = renderAbout(); + else if (parts[0] === "s" && parts[1]) view = await renderSuite(parts[1]); + else if (parts[0] === "b" && parts[2]) view = await renderBench(parts[1], parts.slice(2).join("/"), query); // a case name may itself contain '/' + else if (parts[0] === "c" && parts[1]) view = await renderCommit(parts[1]); + else view = await renderLanding(); + } catch (e) { + view = h("main", {}, h("h2", {}, "Could not load this view"), h("p", { class: "note" }, String(e.message || e)), h("p", {}, h("a", { href: "#/" }, "back to the overview"))); + } + root.replaceChildren(view); + window.scrollTo(0, 0); +} + +async function init() { + try { const t = localStorage.getItem("theme"); if (t) applyTheme(t); } catch (e) { /* ignore */ } + const root = document.getElementById("app"); + try { + state.manifest = await getJSON("manifest.json", { fresh: true }); + state.v = state.manifest.built_at; + [state.project, state.landing, state.series] = await Promise.all([ + getJSON("project.json").catch(() => ({})), getJSON("views/landing.json"), getJSON("views/series.json")]); + } catch (e) { + root.replaceChildren(h("main", {}, h("section", { class: "hero" }, h("div", { class: "eyebrow" }, "No data"), + h("h2", {}, "This page has not been built yet"), + h("p", {}, "Run ", h("code", {}, "bx view --out site"), " to build the static dashboard, or ", h("code", {}, "bx view"), " to serve it locally."), + h("p", { class: "note" }, String(e.message || e))))); + return; + } + if (state.project.title) document.title = state.project.title; + document.body.prepend(header()); + window.addEventListener("hashchange", route); + await route(); +} +init(); diff --git a/src/benchx/views/static/favicon.svg b/src/benchx/views/static/favicon.svg new file mode 100644 index 00000000..61e07f2d --- /dev/null +++ b/src/benchx/views/static/favicon.svg @@ -0,0 +1,6 @@ + + + + + + diff --git a/src/benchx/views/static/index.html b/src/benchx/views/static/index.html new file mode 100644 index 00000000..948c6828 --- /dev/null +++ b/src/benchx/views/static/index.html @@ -0,0 +1,18 @@ + + + + + + +Benchmarks + + + + + + + +

loading…

+ + + diff --git a/src/benchx/views/static/logo.svg b/src/benchx/views/static/logo.svg new file mode 100644 index 00000000..340eec78 --- /dev/null +++ b/src/benchx/views/static/logo.svg @@ -0,0 +1,7 @@ + + benchx + + + + + diff --git a/src/benchx/views/static/style.css b/src/benchx/views/static/style.css new file mode 100644 index 00000000..5765b1dd --- /dev/null +++ b/src/benchx/views/static/style.css @@ -0,0 +1,216 @@ +/* Tokens follow the benchx page contract: light by default, dark under the + OS setting unless the viewer stamped light, and under the viewer's dark stamp. */ +:root { + color-scheme: light; + --page: #f9f9f7; --surface: #fcfcfb; --surface-2: #f0efec; + --ink: #0b0b0b; --ink-2: #4a4946; --muted: #6b6a65; + --grid: #e1e0d9; --axis: #c3c2b7; --border: rgba(11,11,11,0.10); + --series-1: #2a78d6; --series-2: #eb6834; --series-3: #1baf7a; --series-4: #eda100; + --series-5: #e87ba4; --series-6: #008300; --series-7: #6250d6; --series-8: #e34948; + --good: #0ca30c; --good-text: #006300; --warning: #fab219; --serious: #ec835a; --critical: #c42f2f; + --link: #1c5cab; --focus: #2a78d6; --brand: #2a78d6; + --good-bg: rgba(12,163,12,0.10); --bad-bg: rgba(208,59,59,0.09); --warn-bg: rgba(250,178,25,0.14); +} +@media (prefers-color-scheme: dark) { + :root:not([data-theme="light"]) { + color-scheme: dark; + --page: #0d0d0d; --surface: #1a1a19; --surface-2: #262624; + --ink: #f0efec; --ink-2: #c3c2b7; --muted: #898781; + --grid: #2c2c2a; --axis: #383835; --border: rgba(255,255,255,0.10); + --series-1: #3987e5; --series-2: #d95926; --series-3: #199e70; --series-4: #c98500; + --series-5: #d55181; --series-6: #008300; --series-7: #9085e9; --series-8: #e66767; + --good-text: #3fbf3f; --link: #6da7ec; --critical: #ef6a6a; --muted: #9a9890; + --good-bg: rgba(63,191,63,0.12); --bad-bg: rgba(239,106,106,0.12); --warn-bg: rgba(250,178,25,0.12); + } +} +:root[data-theme="dark"] { + color-scheme: dark; + --page: #0d0d0d; --surface: #1a1a19; --surface-2: #262624; + --ink: #f0efec; --ink-2: #c3c2b7; --muted: #898781; + --grid: #2c2c2a; --axis: #383835; --border: rgba(255,255,255,0.10); + --series-1: #3987e5; --series-2: #d95926; --series-3: #199e70; --series-4: #c98500; + --series-5: #d55181; --series-6: #008300; --series-7: #9085e9; --series-8: #e66767; + --good-text: #3fbf3f; --link: #6da7ec; --critical: #ef6a6a; --muted: #9a9890; + --good-bg: rgba(63,191,63,0.12); --bad-bg: rgba(239,106,106,0.12); --warn-bg: rgba(250,178,25,0.12); +} + +* { box-sizing: border-box; } +html { background: var(--page); } +body { margin: 0; background: var(--page); color: var(--ink); font: 15px/1.45 system-ui, -apple-system, "Segoe UI", sans-serif; } +a { color: var(--link); text-decoration: none; } +a:hover { text-decoration: underline; } +code, .mono { font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; font-size: 0.92em; } +.num { font-variant-numeric: tabular-nums; } +:focus-visible { outline: 2px solid var(--focus); outline-offset: 2px; } + +header.top { display: flex; flex-wrap: wrap; gap: 12px 20px; align-items: center; padding: 12px 16px; border-bottom: 1px solid var(--border); background: var(--surface); position: sticky; top: 0; z-index: 5; } +header.top h1 { font-size: 17px; margin: 0; font-weight: 650; } +header.top h1 a { color: var(--ink); } +header.top h1 .brand { display: inline-flex; align-items: center; gap: 9px; } +header.top h1 .brand:hover { text-decoration: none; } +.mark { width: 26px; height: 26px; flex: none; display: block; } +.benchx { display: inline-flex; align-items: center; gap: 4px; color: var(--ink); vertical-align: middle; } +.benchx .wm { font-weight: 700; letter-spacing: -0.02em; } +.benchx .wm b { color: var(--brand); font-weight: 700; } +footer .made { display: inline-flex; align-items: center; gap: 6px; flex-wrap: wrap; } +footer .benchx .mark { width: 16px; height: 16px; } +footer .benchx .wm { font-size: 13px; } +.about-brand .mark { width: 34px; height: 34px; } +.about-brand .benchx { gap: 8px; } +.about-brand .wm { font-size: 24px; } +.about-brand { margin: 4px 0 2px; } +header.top .sub { color: var(--muted); font-size: 13px; } +header.top nav { display: flex; gap: 4px; font-size: 14px; } +header.top nav a { padding: 4px 8px; border-radius: 6px; color: var(--ink-2); } +header.top nav a[aria-current] { background: var(--surface-2); color: var(--ink); font-weight: 600; } +header.top nav a:hover { text-decoration: none; background: var(--surface-2); } +header.top .grow { flex: 1 1 160px; } +.search { position: relative; flex: 1 1 260px; max-width: 520px; } +.search input { width: 100%; padding: 7px 10px; border: 1px solid var(--axis); border-radius: 6px; background: var(--surface); color: var(--ink); font: inherit; } +.search ul { position: absolute; left: 0; right: 0; top: 100%; margin: 4px 0 0; padding: 4px 0; list-style: none; background: var(--surface); border: 1px solid var(--border); border-radius: 6px; box-shadow: 0 6px 24px rgba(0,0,0,0.12); max-height: 60vh; overflow: auto; } +.search li { padding: 6px 10px; cursor: pointer; display: flex; gap: 8px; justify-content: space-between; } +.search li:hover, .search li[aria-selected="true"] { background: var(--surface-2); } +.search li .where { color: var(--muted); font-size: 12px; } +button.ghost { border: 1px solid var(--axis); background: var(--surface); color: var(--ink-2); border-radius: 6px; padding: 6px 10px; font: inherit; cursor: pointer; } + +main { max-width: 1180px; margin: 0 auto; padding: 20px 16px 60px; } +h2 { font-size: 20px; margin: 28px 0 12px; font-weight: 650; } +h2:first-child { margin-top: 4px; } +h3 { font-size: 15px; margin: 18px 0 8px; font-weight: 650; color: var(--ink-2); } +.crumbs { color: var(--muted); font-size: 13px; margin-bottom: 6px; } +.lead { color: var(--ink-2); margin: 0 0 10px; } +.note { color: var(--muted); font-size: 13px; } + +.tiles { display: grid; grid-template-columns: repeat(auto-fit, minmax(150px, 1fr)); gap: 10px; } +.tile { background: var(--surface); border: 1px solid var(--border); border-radius: 8px; padding: 12px 14px; min-height: 76px; } +.tile .label { font-size: 12px; color: var(--muted); letter-spacing: 0.01em; } +.tile .value { font-size: 24px; font-weight: 650; margin-top: 2px; line-height: 1.15; } +.tile .value.small { font-size: 17px; } +.tile .hint { font-size: 12px; color: var(--ink-2); margin-top: 3px; } + +.cols { display: grid; grid-template-columns: repeat(auto-fit, minmax(min(320px, 100%), 1fr)); gap: 16px; margin-top: 24px; } +.cols.tight { margin-top: 14px; gap: 12px 28px; } +.card { background: var(--surface); border: 1px solid var(--border); border-radius: 8px; padding: 12px 14px; } +.card h3 { margin-top: 2px; } + +table { border-collapse: collapse; width: 100%; font-size: 14px; } +th, td { text-align: left; padding: 6px 8px; border-bottom: 1px solid var(--grid); vertical-align: top; } +th { color: var(--muted); font-weight: 600; font-size: 12px; } +td.r, th.r { text-align: right; } +tr:last-child td { border-bottom: 0; } +.wrap { overflow-x: auto; } + +.delta { display: inline-flex; align-items: center; gap: 4px; white-space: nowrap; } +.delta .glyph { font-size: 11px; } +.delta.regressed { color: var(--critical); } +.delta.improved { color: var(--good-text); } +.delta.flat { color: var(--ink-2); } +.delta.none { color: var(--muted); } +.delta.within { opacity: 0.6; font-weight: 400; font-style: italic; } +.delta.within .glyph { opacity: 0.7; } +.delta.unknown { text-decoration: underline dotted; text-underline-offset: 3px; cursor: help; } +.delta-cell { display: inline-flex; gap: 6px; align-items: baseline; } +.tip .row { flex-wrap: wrap; } +.tip .spread { flex-basis: 100%; padding-left: 22px; font-size: 12px; } +.tag { display: inline-block; font-size: 11px; padding: 1px 6px; border-radius: 10px; border: 1px solid var(--border); color: var(--ink-2); background: var(--surface-2); } +.tag.gap { border-color: var(--warning); background: var(--warn-bg); cursor: help; } +.tag.failed { border-color: var(--critical); color: var(--critical); background: var(--bad-bg); } + +.filters { display: flex; flex-wrap: wrap; gap: 10px 16px; align-items: center; margin: 8px 0 12px; font-size: 14px; } +.filters label { display: inline-flex; gap: 6px; align-items: center; color: var(--ink-2); } +.filters select, .filters input { font: inherit; padding: 4px 8px; border: 1px solid var(--axis); border-radius: 6px; background: var(--surface); color: var(--ink); } + +.chart-wrap { background: var(--surface); border: 1px solid var(--border); border-radius: 8px; padding: 12px 8px 4px; position: relative; } +.chart-title { font-size: 14px; color: var(--ink-2); padding: 0 8px 6px; display: flex; justify-content: space-between; gap: 12px; flex-wrap: wrap; } +.chart { width: 100%; } +.tip { position: absolute; pointer-events: none; background: var(--surface); color: var(--ink); border: 1px solid var(--border); border-radius: 6px; box-shadow: 0 6px 20px rgba(0,0,0,0.14); padding: 8px 10px; font-size: 13px; min-width: 180px; max-width: 360px; z-index: 3; display: none; } +.tip .when { color: var(--muted); font-size: 12px; margin-bottom: 4px; } +.tip .subj { color: var(--ink-2); font-size: 12px; margin-top: 4px; white-space: normal; } +.tip .row { display: flex; gap: 8px; align-items: baseline; } +.tip .key { width: 14px; height: 0; border-top: 2px solid; display: inline-block; flex: none; } +.tip .v { font-weight: 650; } +.tip .s { color: var(--ink-2); } + +/* uPlot chrome */ +.uplot { font: inherit !important; } +.uplot .u-legend { font-size: 13px; color: var(--ink-2); text-align: left; padding: 6px 8px 0; } +.uplot .u-legend .u-marker { border-radius: 2px; } +.uplot .u-select { background: rgba(42,120,214,0.12); } +.uplot .u-axis { color: var(--muted); } + +.empty { color: var(--muted); padding: 24px; text-align: center; } +details.table-view summary { cursor: pointer; color: var(--ink-2); font-size: 14px; margin: 10px 0; } +footer { color: var(--muted); font-size: 12px; margin-top: 40px; border-top: 1px solid var(--border); padding-top: 12px; } + +.bad { color: var(--critical); font-weight: 600; } +.button { display: inline-block; padding: 7px 12px; border-radius: 6px; border: 1px solid var(--axis); background: var(--surface); color: var(--ink); font-size: 14px; white-space: nowrap; } +.button:hover { text-decoration: none; border-color: var(--ink-2); } +.banner { border-radius: 8px; padding: 10px 14px; margin: 0 0 14px; font-size: 14px; border: 1px solid var(--border); } +.banner.bad { background: var(--bad-bg); border-color: var(--critical); } +.more { margin: 12px 0 4px; } +.inventory { margin: 12px 2px 0; } + +/* latest run, at the top of the overview */ +.hero { background: var(--surface); border: 1px solid var(--border); border-left: 4px solid var(--axis); border-radius: 10px; padding: 16px 18px; margin-bottom: 12px; } +.hero.good { border-left-color: var(--good); } +.hero.warn { border-left-color: var(--warning); } +.hero.bad { border-left-color: var(--critical); } +.hero h2 { margin: 2px 0 4px; font-size: 19px; line-height: 1.3; } +.hero pre { background: var(--surface-2); border-radius: 6px; padding: 10px 12px; overflow-x: auto; font-size: 13px; } +.hero-head { display: flex; justify-content: space-between; gap: 12px 20px; align-items: flex-start; flex-wrap: wrap; } +.eyebrow { font-size: 12px; font-weight: 600; letter-spacing: 0.04em; text-transform: uppercase; color: var(--muted); } +.chips { display: flex; flex-wrap: wrap; gap: 8px; margin-top: 14px; } +.chip { display: inline-flex; align-items: baseline; gap: 2px; font-size: 13px; color: var(--ink-2); background: var(--surface-2); border-radius: 999px; padding: 3px 11px; } +.chip b { font-size: 15px; color: var(--ink); } +.chip.bad { background: var(--bad-bg); } .chip.bad b { color: var(--critical); } +.chip.good { background: var(--good-bg); } .chip.good b { color: var(--good-text); } +.movers h3 { margin: 4px 0 6px; } +.movers ol { list-style: none; margin: 0; padding: 0; } +.movers li { display: grid; grid-template-columns: minmax(0, 1fr) auto auto; gap: 4px 12px; align-items: baseline; padding: 5px 0; border-bottom: 1px solid var(--grid); font-size: 14px; } +.movers li:last-child { border-bottom: 0; } +.movers .name { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +.movers .from-to { font-size: 12px; } +details.failures { margin-top: 12px; border-top: 1px solid var(--grid); padding-top: 8px; } +details.failures summary { cursor: pointer; font-size: 14px; } +details.failures ul { margin: 8px 0 0; padding-left: 18px; columns: 2 320px; font-size: 14px; } +details.failures li { break-inside: avoid; padding: 2px 0; } +code.reason, .banner code, td code { font-size: 12px; background: var(--surface-2); padding: 1px 5px; border-radius: 4px; } + +/* benchmark cards (asv-style summary grid) */ +.grid { display: grid; grid-template-columns: repeat(auto-fill, minmax(min(250px, 100%), 1fr)); gap: 10px; } +.mini { display: flex; flex-direction: column; background: var(--surface); border: 1px solid var(--border); border-radius: 8px; padding: 10px 11px 7px; color: var(--ink); min-width: 0; } +.mini:hover { text-decoration: none; border-color: var(--ink-2); } +.mini.failed { border-left: 3px solid var(--critical); } +.mini-head { display: flex; justify-content: space-between; gap: 8px; align-items: flex-start; font-size: 13px; } +.mini-head .name { display: -webkit-box; -webkit-line-clamp: 2; -webkit-box-orient: vertical; overflow: hidden; overflow-wrap: anywhere; min-width: 0; line-height: 1.3; } +.mini-head .delta { font-weight: 600; } +.mini-value { display: flex; gap: 8px; align-items: baseline; margin: 3px 0 2px; flex-wrap: wrap; } +.mini-value .num { font-size: 16px; font-weight: 650; } +.mini-value .note { font-size: 12px; } +.spark { width: 100%; height: 52px; margin-top: auto; } +.spark-empty { height: 52px; display: flex; align-items: center; color: var(--muted); font-size: 12px; } +.mini-foot { display: flex; justify-content: space-between; gap: 8px; font-size: 11.5px; color: var(--muted); margin-top: 2px; } +.legend { display: inline-flex; flex-wrap: wrap; } +.legend-key { display: inline-flex; align-items: center; gap: 6px; margin-right: 12px; font-size: 13px; color: var(--ink-2); } +.legend-key .key { width: 14px; border-top: 2px solid; display: inline-block; } +.uplot .u-over, .uplot .u-under { border-radius: 4px; } +h2.title { overflow-wrap: anywhere; margin-bottom: 2px; } + +/* about */ +.prose dl { margin: 0; } +.prose dt { font-weight: 650; margin-top: 14px; } +.prose dd { margin: 4px 0 0; color: var(--ink-2); max-width: 75ch; } +.prose ol { padding-left: 20px; margin: 6px 0 0; } +.prose li { margin: 4px 0; } + +footer { display: flex; flex-wrap: wrap; justify-content: space-between; gap: 6px 16px; } + +@media (max-width: 600px) { + .tile .value { font-size: 20px; } + header.top { padding: 10px 16px; gap: 8px 12px; } + header.top .search { order: 3; flex-basis: 100%; max-width: none; } + .hero { padding: 14px; } + .hero h2 { font-size: 17px; } + .movers li { grid-template-columns: minmax(0, 1fr) auto; } + .movers .from-to { grid-column: 1 / -1; grid-row: 2; } +} diff --git a/tests/conftest.py b/tests/conftest.py index a52dcdf8..2180bd83 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,6 +4,7 @@ compiler record, and a benchmark binary, here the fake from fake_gbench.py. """ +import importlib.util import json import os import shutil @@ -17,6 +18,9 @@ from benchx import runner, snapshot from benchx.core.catalog import SCHEMAS +HAS_PYARROW = importlib.util.find_spec("pyarrow") is not None +requires_pyarrow = pytest.mark.skipif(not HAS_PYARROW, reason="requires pyarrow (benchx[parquet])") + EXAMPLES = SCHEMAS / "measurement-result" / "0.1.0" / "examples" FAKE = Path(__file__).with_name("fake_gbench.py") diff --git a/tests/test_asv.py b/tests/test_asv.py index 9ae425b5..a9167bf6 100644 --- a/tests/test_asv.py +++ b/tests/test_asv.py @@ -11,8 +11,7 @@ from benchx import bench, cli, core, harnesses from benchx.adapters import asv, asv_runtime -from benchx.store import Store -from conftest import warnings_of +from conftest import requires_pyarrow, warnings_of SUITE =textwrap.dedent(''' class Sort: @@ -119,7 +118,9 @@ def test_asv_target_reads_the_benchmark_dir(suite, tmp): bench.asv_target(tmp / "missing") +@requires_pyarrow def test_asv_command_runs_every_time_benchmark(suite, capsys): + from benchx.store import Store assert cli.main([*ASV, "--run-key", "all", "--project", "p", *FAST]) == cli.EXIT_OK err = capsys.readouterr().err assert "skipped module benchmarks.needs_missing" in err diff --git a/tests/test_bench.py b/tests/test_bench.py index 221fb2aa..ab786eb2 100644 --- a/tests/test_bench.py +++ b/tests/test_bench.py @@ -11,8 +11,7 @@ from benchx import cli, config, core, harnesses, session from benchx.errors import SessionError -from benchx.store import Store -from conftest import SOURCE_URI +from conftest import SOURCE_URI, requires_pyarrow MODULE = textwrap.dedent(''' def bench_a(): @@ -68,7 +67,9 @@ def test_both_calibration_flags_conflict(): harnesses.PYPERF.flags_to_precision(min_time=0.1, loops=5) +@requires_pyarrow def test_bench_command_runs_and_ingests(tree, capsys): + from benchx.store import Store assert cli.main(["bench", "mod.py", "bench_a", "--run-key", "cli", "--project", "p", *FAST]) == cli.EXIT_OK out = capsys.readouterr() assert "bench_a" in out.out and "success" in out.out and "bench_b" not in out.out diff --git a/tests/test_bench_harnesses.py b/tests/test_bench_harnesses.py index ad41f72f..8f7da2a1 100644 --- a/tests/test_bench_harnesses.py +++ b/tests/test_bench_harnesses.py @@ -12,8 +12,7 @@ import pytest from benchx import bench, cli, core -from benchx.store import Store -from conftest import EXAMPLES, QUIET, NOISY, cases, git, make_build +from conftest import EXAMPLES, QUIET, NOISY, cases, git, make_build, requires_pyarrow from test_config import write_config from test_target import PROVIDER, describe @@ -30,7 +29,9 @@ def order_of(tmp, run_key): return json.loads((tmp / "results" / run_key / "orders" / "order.json").read_text()) +@requires_pyarrow def test_one_google_benchmark_run_prints_a_table_and_delivers(build, tmp, capsys): + from benchx.store import Store assert cli.main(["bench", str(build), *GBENCH, "--run-key", "g1"]) == cli.EXIT_OK out = capsys.readouterr() assert QUIET in out.out and NOISY in out.out and "success" in out.out @@ -43,6 +44,7 @@ def test_one_google_benchmark_run_prints_a_table_and_delivers(build, tmp, capsys assert {d["coordinates"]["workload"]["name"] for d in Store(tmp / "store.parquet").documents("g1")} == {QUIET, NOISY} +@requires_pyarrow def test_head_table_prints_the_bench_columns_of_the_stored_results(build, capsys): assert cli.main(["bench", str(build), *GBENCH, "--run-key", "h1"]) == cli.EXIT_OK capsys.readouterr() @@ -63,6 +65,7 @@ def test_a_table_row_needs_nothing_the_schema_leaves_optional(example): assert (row["loops"] is None) == ("inner_iterations" not in example.get("procedure", {})) +@requires_pyarrow def test_head_table_shows_results_from_other_producers(tmp, capsys): """The schema's own examples, one of which has no `procedure`, are ingested and tabulated.""" assert cli.main(["ingest", str(EXAMPLES)]) == cli.EXIT_OK @@ -72,6 +75,7 @@ def test_head_table_shows_results_from_other_producers(tmp, capsys): assert "truncate" in out and "parquet-read" in out # narwhals-overhead-ratio.json has no procedure +@requires_pyarrow def test_head_filters_by_project_and_workload_before_taking_n(build, capsys): assert cli.main(["bench", str(build), *GBENCH, "--run-key", "f1", "--project", "alpha"]) == cli.EXIT_OK assert cli.main(["bench", str(build), *GBENCH, "--run-key", "f2", "--project", "beta"]) == cli.EXIT_OK @@ -130,12 +134,14 @@ def test_what_does_not_fit_is_refused_before_anything_runs(argv, message, build, def test_a_suite_the_adapter_cannot_find_is_refused(build, tmp, capsys): - assert cli.main(["bench", str(build), "--adapter", "google-benchmark", "--suite", "gone"]) == cli.EXIT_REFUSED + assert cli.main(["bench", str(build), "--adapter", "google-benchmark", "--suite", "gone", "--no-ingest"]) == cli.EXIT_REFUSED assert "suite binary not found" in capsys.readouterr().err assert not (tmp / "results").exists() +@requires_pyarrow def test_a_provider_target_is_resolved_like_compare_does(repo, build, tmp, monkeypatch, capsys): + from benchx.store import Store descriptions = tmp / "descriptions.json" descriptions.write_text(json.dumps({"plain": describe(build, repo["wt-head"], how_built="cmake plain")})) script = tmp / "proj" / "provider.py" @@ -189,7 +195,9 @@ def test_pyperf_takes_a_working_tree_path_like_the_shorthand(tmp, monkeypatch, c "environment_variables": {"DEMO_THREADS": "1"}} +@requires_pyarrow def test_a_bare_command_takes_everything_from_the_file(build, tmp, capsys): + from benchx.store import Store write_config(tmp, **CONFIG) assert cli.main(["bench", str(build), QUIET, "--run-key", "f1"]) == cli.EXIT_OK assert QUIET in capsys.readouterr().out @@ -213,7 +221,9 @@ def test_a_flag_wins_over_the_file(build, tmp): assert order["environment_variables"] == {"DEMO_THREADS": "2", "OTHER": "x"} +@requires_pyarrow def test_no_project_makes_the_run_ad_hoc(build, tmp): + from benchx.store import Store write_config(tmp, **CONFIG) assert cli.main(["bench", str(build), "--no-project", "--run-key", "f3"]) == cli.EXIT_OK assert "project" not in order_of(tmp, "f3") @@ -238,10 +248,11 @@ def test_the_files_adapter_is_the_only_one_its_settings_fit(build, tmp, capsys): def test_a_quantity_the_harness_does_not_report_is_refused(build, tmp, capsys): - assert cli.main(["bench", str(build), *GBENCH, "--quantity", "peak-rss"]) == cli.EXIT_REFUSED + assert cli.main(["bench", str(build), *GBENCH, "--quantity", "peak-rss", "--no-ingest"]) == cli.EXIT_REFUSED assert "does not report peak-rss" in capsys.readouterr().err +@requires_pyarrow def test_paths_come_from_the_file(build, tmp): write_config(tmp, adapter=CONFIG["adapter"], paths={"store": "data/s.parquet", "results": "out"}) (tmp / "data").mkdir() @@ -264,7 +275,9 @@ def test_a_provider_target_without_a_provider_is_refused(build, tmp, capsys): assert '@plain needs "provider"' in capsys.readouterr().err +@requires_pyarrow def test_compare_takes_its_policy_from_the_file(repo, tmp, capsys): + from benchx.store import Store base = make_build(tmp / "b0", repo["wt-base"], cases()) head = make_build(tmp / "b1", repo["wt-head"], cases(quiet=1.02e-3)) write_config(tmp, adapter=CONFIG["adapter"], compare={"profile": "revisions", "rounds": 3, "k": 4.0}) diff --git a/tests/test_benchmark_source.py b/tests/test_benchmark_source.py index 8f5d301e..917a0784 100644 --- a/tests/test_benchmark_source.py +++ b/tests/test_benchmark_source.py @@ -9,8 +9,7 @@ pytest.importorskip("pyperf") from benchx import adapters, cli, core, runner, snapshot -from benchx.store import Store -from conftest import git +from conftest import git, requires_pyarrow SUITE = textwrap.dedent(''' def bench_a(): @@ -101,6 +100,7 @@ def test_a_new_file_in_a_repository_is_dirty(subject, versioned_suite): assert snapshot.benchmark_identity(new, subject)["dirty"] == "dirty" +@requires_pyarrow def test_an_unversioned_suite_is_recorded_without_a_revision(subject, loose_suite): order = bench(subject, loose_suite, "u1") assert order["benchmark"] == {"source": {"uri": loose_suite.parent.as_uri(), "type": "directory"}} @@ -114,6 +114,7 @@ def test_an_unversioned_suite_is_recorded_without_a_revision(subject, loose_suit assert provenance["subject_dirty"] == "clean" and provenance["subject_tree"] != provenance["benchmark_tree"] +@requires_pyarrow def test_a_versioned_suite_is_recorded_with_its_own_revision(subject, versioned_suite): bench(subject, versioned_suite, "v1") document = result("v1") @@ -123,7 +124,9 @@ def test_a_versioned_suite_is_recorded_with_its_own_revision(subject, versioned_ assert document["revision"]["key"] == git(subject, "rev-parse", "HEAD") +@requires_pyarrow def test_editing_the_suite_changes_its_tree_but_not_the_series(subject, loose_suite, tmp): + from benchx.store import Store bench(subject, loose_suite, "e1") before = result("e1") loose_suite.write_text(SUITE + "\n# edited\n") @@ -137,6 +140,7 @@ def test_editing_the_suite_changes_its_tree_but_not_the_series(subject, loose_su assert len(store.history(series[0]["fingerprint"])) == 2 # both runs are points of it +@requires_pyarrow def test_a_suite_inside_the_subject_is_recorded_as_before(subject): (subject / "bench_in.py").write_text(SUITE) git(subject, "add", "-A") @@ -148,6 +152,7 @@ def test_a_suite_inside_the_subject_is_recorded_as_before(subject): assert document["provenance"]["benchmark_tree"] == document["provenance"]["subject_tree"] +@requires_pyarrow def test_an_order_naming_the_wrong_benchmark_is_refused(subject, loose_suite, tmp): order = bench(subject, loose_suite, "m1") order["benchmark"] = {"source": {"uri": "https://example.org/elsewhere.git", "type": "git"}, "revision": "abc"} diff --git a/tests/test_benchmark_version.py b/tests/test_benchmark_version.py index c71ee9d0..9df14bf3 100644 --- a/tests/test_benchmark_version.py +++ b/tests/test_benchmark_version.py @@ -160,6 +160,7 @@ def test_an_asv_result_has_its_own_version_and_no_pyperf_one(tmp_path): def compare(tmp_path, module, run_key, names=("f",)): pytest.importorskip("pyperf") + pytest.importorskip("pyarrow") from benchx import harnesses, session from benchx.store import Store from test_compare_harnesses import make_worktrees diff --git a/tests/test_build_import_path.py b/tests/test_build_import_path.py index 1f0f0b48..14ed694a 100644 --- a/tests/test_build_import_path.py +++ b/tests/test_build_import_path.py @@ -21,6 +21,8 @@ import pytest +pytest.importorskip("pyarrow") + from benchx import harnesses, session from benchx.store import Store from conftest import SOURCE_URI, git diff --git a/tests/test_collectors.py b/tests/test_collectors.py index 45879a9a..d6c2f747 100644 --- a/tests/test_collectors.py +++ b/tests/test_collectors.py @@ -178,7 +178,7 @@ def result_of(run_key): def test_bench_runs_the_collector_and_records_its_facts(project): configure(project) - assert cli.main(["bench", "@x", *FAST, "--run-key", "c1"]) == cli.EXIT_OK + assert cli.main(["bench", "@x", *FAST, "--run-key", "c1", "--no-ingest"]) == cli.EXIT_OK document = result_of("c1") assert core.check(document, "measurement-result") == [] assert document["coordinates"]["environment"]["system"]["facts"]["s"] == 1 and "sha256" in document["coordinates"]["environment"]["system"]["facts"] @@ -190,7 +190,7 @@ def test_bench_runs_the_collector_and_records_its_facts(project): def test_a_mismatch_refuses_the_order_before_any_case(project, capsys, monkeypatch): configure(project) monkeypatch.setenv("K", "2") - assert cli.main(["bench", "@x", *FAST, "--run-key", "c2"]) == cli.EXIT_REFUSED + assert cli.main(["bench", "@x", *FAST, "--run-key", "c2", "--no-ingest"]) == cli.EXIT_REFUSED assert "how_built /k: declared 1, observed 2" in capsys.readouterr().err assert not list(project.glob("results/c2/*_wall-time.json")) @@ -198,7 +198,7 @@ def test_a_mismatch_refuses_the_order_before_any_case(project, capsys, monkeypat def test_a_warn_mismatch_runs_and_marks_every_result(project, monkeypatch): configure(project, on_fail="warn") monkeypatch.setenv("K", "2") - assert cli.main(["bench", "@x", *FAST, "--run-key", "c3"]) == cli.EXIT_OK + assert cli.main(["bench", "@x", *FAST, "--run-key", "c3", "--no-ingest"]) == cli.EXIT_OK assert warnings_of(result_of("c3")) == ["collector-mismatch"] @@ -206,7 +206,7 @@ def test_bx_run_refuses_an_order_with_collectors_unless_allowed(project, tmp, ca configure(project) assert cli.main(["bench", "@x", *FAST, "--run-key", "c4", "--no-ingest"]) == cli.EXIT_OK order = project / "results" / "c4" / "orders" / "order.json" - assert cli.main(["run", str(order), "--out", str(tmp / "o1")]) == cli.EXIT_REFUSED + assert cli.main(["run", str(order), "--out", str(tmp / "o1"), "--no-ingest"]) == cli.EXIT_REFUSED assert "--allow-collectors" in capsys.readouterr().err with pytest.raises(runner.Refused): runner.run(order, tmp / "o2") diff --git a/tests/test_compare_harnesses.py b/tests/test_compare_harnesses.py index 0408423b..e14b534b 100644 --- a/tests/test_compare_harnesses.py +++ b/tests/test_compare_harnesses.py @@ -13,8 +13,8 @@ import pytest from benchx import cli, core, harnesses, runner, session -from benchx.store import Store -from conftest import SOURCE_URI, git +from benchx.results import MemoryStore +from conftest import SOURCE_URI, git, requires_pyarrow MEASURE = ["A", "B", "--profile", "revisions", "--suite", "s", "--rounds", "1", "--adapter", "pyperf"] @@ -42,7 +42,7 @@ def test_a_harness_refusal_comes_before_any_side_is_resolved(tmp): # "nowhere" does not exist: the flag is refused first, so that is never looked at. with pytest.raises(session.SessionError, match="pyperf takes no --repetitions"): session.compare_targets("nowhere", "nowhere", profile="revisions", suite="s", rounds=1, - store=Store(tmp / "s.parquet"), adapter="pyperf", flags={"repetitions": 3}) + store=MemoryStore(), adapter="pyperf", flags={"repetitions": 3}) def test_a_pyperf_side_order(tmp): @@ -114,7 +114,9 @@ def param_worktrees(tmp_path): return make_worktrees(tmp_path, PARAM_MODULE) +@requires_pyarrow def test_pyperf_revisions_end_to_end(tmp, worktrees, monkeypatch): + from benchx.store import Store pytest.importorskip("pyperf") def provider(spec, operation, provider=None): # what a minimal provider for two worktrees would answer @@ -136,8 +138,10 @@ def provider(spec, operation, provider=None): # what a minimal provider for two assert order["target"]["kind"] == "working_tree" +@requires_pyarrow def test_each_combination_of_a_function_with_params_is_a_unit(tmp, param_worktrees): """Two combinations are two units, each paired with the same combination on the other side.""" + from benchx.store import Store pytest.importorskip("pyperf") doc = session.compare_targets( str(param_worktrees["base"]), str(param_worktrees["head"]), profile="revisions", suite="mod.py", rounds=2, @@ -148,9 +152,11 @@ def test_each_combination_of_a_function_with_params_is_a_unit(tmp, param_worktre assert doc["missing"] == [] and doc["failed_invariants"] == [] +@requires_pyarrow def test_a_filter_that_selects_no_case_refuses_the_comparison(tmp, worktrees): """A comparison with nothing measured on a side is not "no difference": it is refused with the side's own refusal, before any case runs, and nothing reaches the store.""" + from benchx.store import Store pytest.importorskip("pyperf") store = Store(tmp / "s.parquet") with pytest.raises(runner.Refused, match=r"filter '\^\(nosuch\)\$' selects no case in mod.py"): @@ -161,8 +167,10 @@ def test_a_filter_that_selects_no_case_refuses_the_comparison(tmp, worktrees): assert list(store.documents("empty")) == [] +@requires_pyarrow def test_pyperf_bare_paths_are_working_trees(tmp, worktrees): """Without a provider, a bare path is a working tree for pyperf (as it is a build for Google Benchmark).""" + from benchx.store import Store pytest.importorskip("pyperf") doc = session.compare_targets( str(worktrees["base"]), str(worktrees["head"]), profile="revisions", suite="mod.py", rounds=2, @@ -216,6 +224,8 @@ def variants_tree(tmp, source, files=None): def run_variants(tmp, tree, **kwargs): """`session.compare_variants` on `tree`: the document, the store, and what the session said.""" + pytest.importorskip("pyarrow") + from benchx.store import Store store, said = Store(tmp / "s.parquet"), [] settings = {"suite": "mod.py", "rounds": 2, "min_rounds": 2, "adapter": "pyperf", "flags": FAST, "out": tmp / "out", "run_key": "v", **kwargs} @@ -255,7 +265,9 @@ def test_variants_run_one_order_per_value_and_round_with_that_values_cases_only( "round 2/2 role='variant'"] +@requires_pyarrow def test_variants_the_baseline_is_the_first_declared_value_and_can_be_swapped(tmp): + from benchx.store import Store pytest.importorskip("pyperf") tree = variants_tree(tmp, VARIANTS) doc, _, said = run_variants(tmp, tree) @@ -270,7 +282,9 @@ def test_variants_the_baseline_is_the_first_declared_value_and_can_be_swapped(tm "coordinates"]["workload"]["parameters"] == {"role": "'variant'"} +@requires_pyarrow def test_variants_an_environment_variable_is_a_second_parameter(tmp): + from benchx.store import Store pytest.importorskip("pyperf") tree, store = variants_tree(tmp, VARIANTS), Store(tmp / "s.parquet") with pytest.raises(session.SessionError, match=r"several workload parameters: OMP_NUM_THREADS \(-e\), role " @@ -314,7 +328,9 @@ def test_variants_a_benchmark_without_the_parameter_runs_on_neither_side_and_is_ (VARIANTS, {"baseline": "'varient'"}, "baseline \"'varient'\" is not a value of role; use one of: 'baseline', 'variant'"), ], ids=["no-parameter", "unknown-parameter", "three-values", "benchmarks-disagree", "filter-leaves-one-value", "unquoted-baseline", "mistyped-baseline"]) +@requires_pyarrow def test_variants_refusals_come_before_anything_runs(tmp, source, kwargs, message): + from benchx.store import Store pytest.importorskip("pyperf") tree, store = variants_tree(tmp, source), Store(tmp / "s.parquet") with pytest.raises(session.SessionError, match=message): @@ -324,7 +340,7 @@ def test_variants_refusals_come_before_anything_runs(tmp, source, kwargs, messag def test_variants_google_benchmark_has_no_parameters_to_vary(tmp): with pytest.raises(session.SessionError, match="variants needs a parameter the benchmark declares"): - session.compare_variants("nowhere", suite="s", rounds=1, store=Store(tmp / "s.parquet"), + session.compare_variants("nowhere", suite="s", rounds=1, store=MemoryStore(), adapter="google-benchmark") diff --git a/tests/test_discovery_timeout.py b/tests/test_discovery_timeout.py index c3d734a0..50e94386 100644 --- a/tests/test_discovery_timeout.py +++ b/tests/test_discovery_timeout.py @@ -96,7 +96,7 @@ def test_a_binary_that_never_lists_its_benchmarks_is_a_refusal(repo, tmp, capsys assert message in str(raised.value) assert [p for p in (tmp / "out").glob("*.json") if not p.name.startswith("workorder-")] == [] with finishes_within(20): - assert cli.main(["run", str(path), "--out", str(tmp / "out")]) == cli.EXIT_REFUSED + assert cli.main(["run", str(path), "--out", str(tmp / "out"), "--no-ingest"]) == cli.EXIT_REFUSED assert f"refused: {message}" in capsys.readouterr().err diff --git a/tests/test_minimal_deps.py b/tests/test_minimal_deps.py new file mode 100644 index 00000000..187f948f --- /dev/null +++ b/tests/test_minimal_deps.py @@ -0,0 +1,464 @@ +"""Minimal-dependency behavior: optional PyArrow, narrowed format checkers, and RFC 6901 pointers.""" + +import importlib +import json +import subprocess +import sys +from pathlib import Path + +import pytest +from jsonschema import FormatChecker + +from benchx import cli, core, identity +from benchx.core import catalog +from benchx.core.errors import StructureError +from benchx.results import MemoryStore, store_project +from conftest import NOISY, QUIET, cases, git, make_build, order, requires_pyarrow, run_rounds + + +def _block_pyarrow(monkeypatch): + """Simulate an environment without pyarrow, even when pyarrow is installed.""" + import benchx + + for name in list(sys.modules): + if name == "pyarrow" or name.startswith("pyarrow.") or name in ("benchx.store", "benchx.parquet"): + monkeypatch.delitem(sys.modules, name, raising=False) + for attr in ("store", "parquet"): + monkeypatch.delattr(benchx, attr, raising=False) + monkeypatch.setitem(sys.modules, "pyarrow", None) + monkeypatch.setitem(sys.modules, "pyarrow.parquet", None) + + +# --- RFC 6901 error-location formatting (replaces jsonpointer) ------------------------- + +@pytest.mark.parametrize("path, expected", [ + ((), "/"), + (("source", "uri"), "/source/uri"), + (("measurement", "observations", 0), "/measurement/observations/0"), + (("",), "/"), + (("a", "", "b"), "/a//b"), + (("", ""), "//"), + (("a/b", "c~d", "~1/0"), "/a~1b/c~0d/~01~10"), + (("~", "/"), "/~0/~1"), +]) +def test_error_location_rfc6901_pointer(path, expected): + err = StructureError(path=path, keyword="type", reason="bad") + assert err.location == expected + assert str(err) == f"malformed at {expected}: bad" + + +# --- Narrowed JSON Schema format checkers & missing-checker enforcement ---------------- + +@pytest.mark.parametrize("bad_uri", [ + "not a uri", + "relative/path", + "http://[::1", +]) +def test_invalid_uri_rejected_by_format_checker(adhoc, bad_uri): + adhoc["source"]["uri"] = bad_uri + errors = core.check(adhoc, "measurement-result") + assert len(errors) == 1 + assert isinstance(errors[0], StructureError) + assert (errors[0].path, errors[0].keyword) == (("source", "uri"), "format") + + +@pytest.mark.parametrize("bad_date", [ + "2026-02-30T10:02:31Z", + "2026-04-31T12:00:00Z", + "2026-13-01T00:00:00Z", +]) +def test_impossible_calendar_date_rejected_by_format_checker(adhoc, bad_date): + # Matches the UTC Z regex pattern, so only the `date-time` format checker catches it. + adhoc["provenance"]["started_at"] = bad_date + adhoc["provenance"]["ended_at"] = bad_date + errors = core.check(adhoc, "measurement-result") + assert errors + assert all(isinstance(e, StructureError) and e.keyword == "format" for e in errors) + assert {e.path for e in errors} == {("provenance", "started_at"), ("provenance", "ended_at")} + + +def test_valid_leap_and_boundary_dates_accepted(adhoc): + adhoc["provenance"]["started_at"] = "2024-02-29T23:59:59Z" + adhoc["provenance"]["ended_at"] = "2024-02-29T23:59:59Z" + assert core.check(adhoc, "measurement-result") == [] + + +@pytest.mark.parametrize("missing_fmt", ["uri", "date-time"]) +def test_missing_required_format_checker_fails_loudly(monkeypatch, missing_fmt): + """If a required format validator is absent, catalog fails instead of silently skipping.""" + real_checker = FormatChecker() + remaining = tuple(fmt for fmt in real_checker.checkers if fmt != missing_fmt) + monkeypatch.setattr(catalog, "FormatChecker", lambda: FormatChecker(formats=remaining)) + catalog.clear_cache() + try: + with pytest.raises(catalog.FormatCheckerError, match=missing_fmt): + catalog.lookup("measurement-result", catalog.MEASUREMENT_RESULT) + with pytest.raises(catalog.FormatCheckerError, match=missing_fmt): + catalog.fragment("measurement-result", "0.1.0", "/$defs/uri") + finally: + catalog.clear_cache() + + +def test_custom_benchx_schemas_missing_format_checker_fails(tmp_path, monkeypatch): + """Custom schemas under $BENCHX_SCHEMAS fail clearly when a declared format has no checker.""" + base_dir = tmp_path / "schemas" + leaf = base_dir / "custom-leaf" / "0.1.0" + leaf.mkdir(parents=True) + (leaf / "schema.json").write_text(json.dumps({ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:benchx:schema:custom-leaf:0.1.0", + "type": "string", + "format": "no-such-custom-format", + })) + root = base_dir / "measurement-result" / "0.1.0" + root.mkdir(parents=True) + (root / "schema.json").write_text(json.dumps({ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:benchx:schema:measurement-result:0.1.0", + "type": "object", + "properties": { + "x": {"$ref": "urn:benchx:schema:custom-leaf:0.1.0"}, + }, + })) + monkeypatch.setattr(catalog, "SCHEMAS", base_dir) + catalog.clear_cache() + try: + with pytest.raises(catalog.FormatCheckerError, match="no-such-custom-format"): + catalog.lookup("measurement-result", "benchx/measurement-result/0.1.0") + finally: + catalog.clear_cache() + + +# --- Canonical hashes and identities remain unchanged ---------------------------------- + +def test_canonical_hashes_and_identities_unchanged(adhoc, arrow): + assert core.sha256_hex(core.canonical(adhoc)) == ( + "a46dd3a0aff71599c81ef0866c4912b6cd68544e8339a60ff7694c5c62aae720" + ) + assert identity.reported_coordinates_fingerprint(adhoc) == ( + "c6037bbc4eaa0ed27fa54aab68ff1eeb34968cbe15bc89828615223776e0e54d" + ) + assert identity.series_fingerprint(adhoc, store_project(adhoc), identity.MEDIAN) == ( + "b44af1d5029f2b1a222c19cf1acff434e1cb858086aec5c46093e6f9b9d0cd79" + ) + + assert core.sha256_hex(core.canonical(arrow)) == ( + "821864f4131f18e89226f733da8b0246f4275373fe95a50bdd79efc40994fe0c" + ) + assert identity.reported_coordinates_fingerprint(arrow) == ( + "0ca627d985be3f63c29e306939a598e29338d63983d8186ebee5d7c8f6492926" + ) + assert identity.series_fingerprint(arrow, store_project(arrow), identity.MEDIAN) == ( + "fb97ab834dea457cef9af2929daf359512383977e2e811db176899e9f5297221" + ) + + +# --- Arrow-free CLI imports, execution, and JSON-only comparisons ---------------------- + +def test_importing_cli_and_non_store_modules_does_not_import_pyarrow(): + code = ( + "import sys\n" + "import benchx.cli, benchx.bench, benchx.compare, benchx.tables, benchx.results, benchx.runner, benchx.views\n" + "assert 'pyarrow' not in sys.modules, 'pyarrow was imported unexpectedly'\n" + ) + proc = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True, check=False) + assert proc.returncode == 0, proc.stderr + + +def test_cli_help_without_pyarrow(monkeypatch): + _block_pyarrow(monkeypatch) + with pytest.raises(SystemExit) as exc: + cli.main(["--help"]) + assert exc.value.code == 0 + + +def test_bench_run_and_compare_results_without_pyarrow(repo, tmp, monkeypatch, capsys): + """End-to-end without pyarrow: bx run --no-ingest, bx bench --no-ingest, bx compare --results.""" + _block_pyarrow(monkeypatch) + base = make_build(tmp / "build-base", repo["wt-base"], cases()) + head = make_build(tmp / "build-head", repo["wt-head"], cases(quiet=1.02e-3, noisy=2.06e-3)) + + # 1. bx run --no-ingest + order_path = tmp / "order.json" + order_path.write_text(json.dumps(order(base, repo["wt-base"], "run-no-ingest"))) + assert cli.main(["run", str(order_path), "--out", str(tmp / "run-out"), "--no-ingest"]) == cli.EXIT_OK + assert not (tmp / "store.parquet").exists() + capsys.readouterr() + + # 2. bx bench --no-ingest across 3 alternating rounds of two sides (split into two dirs) + out_base = tmp / "out-base" + out_head = tmp / "out-head" + slot = 0 + for r in range(3): + for target, out_dir in ( + (f"{base}:{repo['wt-base']}", out_base), + (f"{head}:{repo['wt-head']}", out_head), + ): + assert cli.main([ + "bench", target, + "--adapter", "google-benchmark", + "--suite", "demo-bench", + "--run-key", "rev-no-arrow", + "--round", str(r), + "--slot", str(slot), + "--out", str(out_dir), + "--no-ingest", + ]) == cli.EXIT_OK + slot += 1 + assert not (tmp / "store.parquet").exists() + capsys.readouterr() + + # 3. bx compare --run ... --results DIR1 DIR2 (JSON and --tables) without pyarrow + assert cli.main([ + "compare", + "--run", "rev-no-arrow", + "--profile", "revisions", + "--baseline", repo["base"][:10], + "--results", str(out_base), str(out_head), + "--json", + ]) == cli.EXIT_OK + doc = json.loads(capsys.readouterr().out) + assert core.check(doc, "comparison-document") == [] + assert doc["failed_invariants"] == [] and doc["missing"] == [] + verdicts = {u["workload"]["name"]: u["verdict"] for u in doc["units"]} + assert verdicts == {QUIET: "regressed", NOISY: "no-change-detected"} + + assert cli.main([ + "compare", + "--run", "rev-no-arrow", + "--profile", "revisions", + "--baseline", repo["base"][:10], + "--results", str(out_base), str(out_head), + "--tables", + ]) == cli.EXIT_OK + table_out = capsys.readouterr().out + assert "REGRESSED" in table_out and "| Change" in table_out + assert not list(tmp.glob("*.parquet")) + + +def test_compare_measure_mode_no_ingest_without_pyarrow(repo, tmp, monkeypatch, capsys): + """`bx compare BASE CONTENDER --no-ingest` runs and compares in memory without pyarrow.""" + _block_pyarrow(monkeypatch) + base = make_build(tmp / "b-base", repo["wt-base"], cases()) + head = make_build(tmp / "b-head", repo["wt-head"], cases(quiet=1.02e-3, noisy=2.06e-3)) + out = tmp / "measure-out" + + assert cli.main([ + "compare", f"{base}:{repo['wt-base']}", f"{head}:{repo['wt-head']}", + "--profile", "revisions", + "--adapter", "google-benchmark", + "--suite", "demo-bench", + "--rounds", "4", + "--run-key", "m-no-arrow", + "--out", str(out), + "--no-ingest", + "--json", + ]) == cli.EXIT_OK + measured = json.loads(capsys.readouterr().out) + assert core.check(measured, "comparison-document") == [] + assert measured["failed_invariants"] == [] and not list(tmp.glob("*.parquet")) + assert {u["workload"]["name"]: u["verdict"] for u in measured["units"]} == { + QUIET: "regressed", NOISY: "no-change-detected", + } + + # Reading those same JSON files back with --results gives the exact same document. + assert cli.main([ + "compare", "--run", "m-no-arrow", "--profile", "revisions", + "--baseline", repo["base"][:10], "--results", str(out), "--json", + ]) == cli.EXIT_OK + assert json.loads(capsys.readouterr().out) == measured + + # --no-ingest is rejected in read mode. + assert cli.main([ + "compare", "--run", "m-no-arrow", "--profile", "revisions", + "--baseline", repo["base"][:10], "--no-ingest", + ]) == cli.EXIT_REFUSED + assert "measure mode" in capsys.readouterr().err + + +def test_compare_variants_measure_mode_no_ingest_without_pyarrow(tmp, monkeypatch, capsys): + """`bx compare TARGET --profile variants --no-ingest` works without pyarrow.""" + pytest.importorskip("pyperf") + _block_pyarrow(monkeypatch) + tree = tmp / "variants-tree" + tree.mkdir() + git(tree, "init", "-q") + (tree / "mod.py").write_text( + "def bench_work(role):\n" + " sum(range(10 if role == 'baseline' else 2000))\n" + "bench_work.params = ['baseline', 'variant']\n" + "bench_work.param_names = ['role']\n" + ) + git(tree, "add", "-A") + git(tree, "commit", "-qm", "init") + + assert cli.main([ + "compare", str(tree), + "--profile", "variants", + "--adapter", "pyperf", + "--suite", "mod.py", + "--rounds", "2", + "--min-rounds", "2", + "--processes", "2", + "--values", "2", + "--loops", "50", + "--warmups", "0", + "--run-key", "v-no-arrow", + "--out", str(tmp / "v-out"), + "--no-ingest", + "--tables", + "--all", + ]) == cli.EXIT_OK + out = capsys.readouterr().out + assert "bench_work" in out and "| Change" in out + assert not list(tmp.glob("*.parquet")) + + +@pytest.mark.parametrize("argv, extra_hint", [ + (["ingest", "results"], None), + (["series"], None), + (["history", "deadbeef"], None), + (["head"], None), + (["run", "order.json"], "--no-ingest"), + (["compare", "--run", "r1", "--profile", "revisions", "--baseline", "abc"], "--results"), + (["compare", "A", "B", "--profile", "revisions", "--adapter", "pyperf", "--suite", "s", "--rounds", "3"], "--no-ingest"), +]) +def test_store_commands_give_actionable_error_without_pyarrow(monkeypatch, capsys, argv, extra_hint): + _block_pyarrow(monkeypatch) + assert cli.main(argv) == cli.EXIT_REFUSED + err = capsys.readouterr().err + assert "benchx[parquet]" in err + if extra_hint: + assert extra_hint in err + + +def test_bench_without_no_ingest_gives_actionable_error_without_pyarrow(repo, tmp, monkeypatch, capsys): + _block_pyarrow(monkeypatch) + base = make_build(tmp / "b", repo["wt-base"], cases()) + assert cli.main([ + "bench", f"{base}:{repo['wt-base']}", + "--adapter", "google-benchmark", + "--suite", "demo-bench", + ]) == cli.EXIT_REFUSED + err = capsys.readouterr().err + assert "benchx[parquet]" in err and "--no-ingest" in err + + +def test_direct_store_and_parquet_imports_explain_parquet_extra(monkeypatch): + _block_pyarrow(monkeypatch) + with pytest.raises(ModuleNotFoundError, match=r"benchx\[parquet\]"): + importlib.import_module("benchx.store") + with pytest.raises(ModuleNotFoundError, match=r"benchx\[parquet\]"): + importlib.import_module("benchx.parquet") + + +# --- Parity between JSON-only (MemoryStore) and Parquet-backed (Store) ----------------- + +@requires_pyarrow +def test_json_only_and_parquet_store_parity_for_comparisons_and_failed_cases(repo, tmp, capsys): + from benchx.store import Store + + base = make_build(tmp / "b-base", repo["wt-base"], cases(), error=["BM_Broken"], skip=["BM_Skipped"]) + head = make_build(tmp / "b-head", repo["wt-head"], cases(quiet=1.02e-3, noisy=2.06e-3), + error=["BM_Broken"], skip=["BM_Skipped"]) + out = run_rounds( + tmp, + [{"build": base, "source": repo["wt-base"]}, {"build": head, "source": repo["wt-head"]}], + rounds=4, + run_key="parity-1", + out=tmp / "results-parity", + ) + + store = Store(tmp / "store.parquet") + assert store.ingest_paths([out])["rejected"] == [] + + # Compare JSON documents from Parquet store vs --results (MemoryStore) + common = ["--run", "parity-1", "--profile", "revisions", "--baseline", repo["base"][:10]] + assert cli.main(["compare", *common, "--json"]) == cli.EXIT_OK + from_parquet = json.loads(capsys.readouterr().out) + + assert cli.main(["compare", *common, "--results", str(out), "--json"]) == cli.EXIT_OK + from_json = json.loads(capsys.readouterr().out) + assert from_json == from_parquet + + # Compare --tables text output too + assert cli.main(["compare", *common, "--tables", "--all"]) == cli.EXIT_OK + tables_parquet = capsys.readouterr().out + assert cli.main(["compare", *common, "--results", str(out), "--tables", "--all"]) == cli.EXIT_OK + tables_json = capsys.readouterr().out + assert tables_json == tables_parquet + + +@requires_pyarrow +def test_json_only_and_parquet_store_parity_for_ingest_contract(repo, tmp, adhoc, capsys): + from benchx.store import Store + + base = make_build(tmp / "b-base", repo["wt-base"], cases()) + head = make_build(tmp / "b-head", repo["wt-head"], cases(quiet=1.02e-3, noisy=2.06e-3)) + out = run_rounds( + tmp, + [{"build": base, "source": repo["wt-base"]}, {"build": head, "source": repo["wt-head"]}], + rounds=3, + run_key="contract-1", + out=tmp / "results-contract", + ) + + parquet_store = Store(tmp / "store.parquet") + memory_store = MemoryStore() + + # Initial ingest and duplicate re-ingest + assert memory_store.ingest_paths([out]) == parquet_store.ingest_paths([out]) + assert memory_store.ingest_paths([out]) == parquet_store.ingest_paths([out]) + assert memory_store.documents("contract-1") == parquet_store.documents("contract-1") + + doc = memory_store.documents("contract-1")[0] + ref = doc["provenance"]["info"]["workorder_ref"] + assert memory_store.order(ref) == parquet_store.order(ref) + + # Every conflict and malformed rejection code matches identically + cases_to_check = [] + + unit_conflict = json.loads(json.dumps(doc)) + unit_conflict["ingest_key"] += ":ms" + unit_conflict["measurement"]["quantity"]["unit"] = "ms" + cases_to_check.append(("unit.json", json.dumps(unit_conflict))) + + attempt_conflict = json.loads(json.dumps(doc)) + attempt_conflict["ingest_key"] += ":moved" + attempt_conflict["coordinates"]["workload"]["name"] = "other" + cases_to_check.append(("attempt.json", json.dumps(attempt_conflict))) + + identity_violation = json.loads(json.dumps(doc)) + identity_violation["ingest_key"] += ":src" + identity_violation["coordinates"]["subject"]["components"][0]["source"] = "https://elsewhere.org/x" + cases_to_check.append(("identity.json", json.dumps(identity_violation))) + + idempotency_conflict = json.loads(json.dumps(doc)) + idempotency_conflict["measurement"]["observations"][0] *= 2 + cases_to_check.append(("idem.json", json.dumps(idempotency_conflict))) + + nan_text = json.dumps(doc).replace('"observations": [', '"observations": [NaN, ', 1) + cases_to_check.append(("nan.json", nan_text)) + + dup_key_text = json.dumps(doc)[:-1] + ', "ingest_key": "other"}' + cases_to_check.append(("dupkey.json", dup_key_text)) + + assert memory_store.ingest(cases_to_check) == parquet_store.ingest(cases_to_check) + + # Unreadable files and bad workorders in a directory sweep + bad_dir = tmp / "bad-sweep" + bad_dir.mkdir() + (bad_dir / "good.json").write_text(json.dumps(adhoc)) + (bad_dir / "invalid-utf8.json").write_bytes(b"\xff\xfe\x00") + (bad_dir / "workorder-bad.json").write_text("{broken") + missing = tmp / "does-not-exist.json" + assert memory_store.ingest_paths([bad_dir, missing]) == parquet_store.ingest_paths([bad_dir, missing]) + + # CLI `--results` reports rejections identically and exits EXIT_INVALID + assert cli.main([ + "compare", "--run", "contract-1", "--profile", "revisions", + "--baseline", repo["base"][:10], "--results", str(bad_dir), + ]) == cli.EXIT_INVALID + err = capsys.readouterr().err + assert "rejected invalid-utf8.json: malformed" in err + assert "rejected workorder-bad.json: malformed" in err diff --git a/tests/test_parquet.py b/tests/test_parquet.py index d5dd701e..8b52f7be 100644 --- a/tests/test_parquet.py +++ b/tests/test_parquet.py @@ -7,11 +7,12 @@ from datetime import datetime from pathlib import Path -import pyarrow as pa -import pyarrow.parquet as pq import pytest import rfc8785 +pa = pytest.importorskip("pyarrow") +import pyarrow.parquet as pq + from benchx import parquet from benchx.core import loader from benchx.core.catalog import SCHEMAS diff --git a/tests/test_prototype.py b/tests/test_prototype.py index 204d77d6..c478885f 100644 --- a/tests/test_prototype.py +++ b/tests/test_prototype.py @@ -10,8 +10,7 @@ from benchx import cli, compare, core, identity, runner from benchx.core.errors import StructureError -from benchx.store import Store -from conftest import NOISY, QUIET, SOURCE_URI, cases, git, make_build, order, run_rounds +from conftest import NOISY, QUIET, SOURCE_URI, cases, git, make_build, order, requires_pyarrow, run_rounds def load(path: Path) -> dict: @@ -73,7 +72,9 @@ def test_results_are_valid_documents(revisions_run): assert load_["attempt_key"] in attempts +@requires_pyarrow def test_revisions_profile(revisions_run, repo, tmp): + from benchx.store import Store store = Store(tmp / "store.parquet") assert store.ingest_paths([revisions_run])["ingested"] == 48 # with the load averages doc = compare.compare(store.documents("rev-1"), run_key="rev-1", profile="revisions", @@ -86,7 +87,9 @@ def test_revisions_profile(revisions_run, repo, tmp): assert all(u["rounds"] == 6 for u in doc["units"]) +@requires_pyarrow def test_environments_profile(environments_run, tmp): + from benchx.store import Store store = Store(tmp / "store.parquet") store.ingest_paths([environments_run]) doc = compare.compare(store.documents("env-1"), run_key="env-1", profile="environments", label="build", @@ -107,7 +110,9 @@ def test_environments_needs_a_real_difference(repo, tmp): assert doc["units"] == [] +@requires_pyarrow def test_ingest_contract(revisions_run, tmp): + from benchx.store import Store store = Store(tmp / "store.parquet") assert store.ingest_paths([revisions_run]) == {"ingested": 48, "duplicate": 0, "rejected": []} # criterion 2: re-ingest is a visible no-op @@ -133,8 +138,10 @@ def attempt(mutate, key_suffix): assert store.ingest([("z.json", text)])["rejected"][0]["code"] == "malformed" +@requires_pyarrow def test_same_document_from_any_store(revisions_run, repo, tmp): """Criterion 8: a throwaway store and a persistent one give one document.""" + from benchx.store import Store documents = [] for name in ("persistent", "throwaway"): store = Store(tmp / f"{name}.parquet") @@ -144,8 +151,10 @@ def test_same_document_from_any_store(revisions_run, repo, tmp): assert documents[0] == documents[1] +@requires_pyarrow def test_traceability(revisions_run, tmp): """Criterion 6: result → order, verdict → results.""" + from benchx.store import Store store = Store(tmp / "store.parquet") store.ingest_paths([revisions_run]) docs = store.documents("rev-1") @@ -187,8 +196,10 @@ def _dirty_run(repo, tmp, profile_sides, run_key): return run_rounds(tmp, profile_sides(base, head), rounds=3, run_key=run_key, out=tmp / run_key) +@requires_pyarrow def test_ad_hoc_is_thin_and_a_dirty_run_is_accepted(repo, tmp): """Criterion 7: no project, no configuration; a dirty single run is recorded as dirty (UC-01).""" + from benchx.store import Store out = _dirty_run(repo, tmp, lambda base, head: [{"build": base, "source": repo["wt-base"]}, {"build": head, "source": repo["wt-head"]}], "adhoc") docs = results(out) @@ -199,9 +210,11 @@ def test_ad_hoc_is_thin_and_a_dirty_run_is_accepted(repo, tmp): assert {s["project"] for s in store.series()} == {f"local/{docs[0]['coordinates']['environment']['identity']['runner']}"} +@requires_pyarrow def test_revisions_labels_a_dirty_side_local_only(repo, tmp): """Criterion 7, UC-03 §10: a revisions comparison with a dirty side is compared and labeled local-only, and the sides that are not clean are named.""" + from benchx.store import Store out = _dirty_run(repo, tmp, lambda base, head: [{"build": base, "source": repo["wt-base"]}, {"build": head, "source": repo["wt-head"]}], "dirty-rev") store = Store(tmp / "store.parquet") @@ -217,8 +230,10 @@ def test_revisions_labels_a_dirty_side_local_only(repo, tmp): assert "[local-only]" in shown and "contender: dirty-subject-tree" in shown +@requires_pyarrow def test_environments_labels_a_dirty_side_local_only(repo, tmp): """Criterion 7, schema §5.5: the environments profile still allows a dirty side, labeled.""" + from benchx.store import Store out = _dirty_run(repo, tmp, lambda base, head: [ {"build": base, "source": repo["wt-head"], "labels": {"build": "a"}}, {"build": make_build(tmp / "b3", repo["wt-head"], cases(quiet=1.02e-3), hardened=True), @@ -267,8 +282,10 @@ def test_a_single_run_records_an_unknown_tree_without_a_tree_id(repo, tmp): assert "subject_tree" not in d["provenance"] and "benchmark_tree" not in d["provenance"] +@requires_pyarrow def test_head_and_history_show_the_dirty_warnings(repo, tmp, capsys): """Q7: the listings mark a dirty result and leave a clean one unmarked.""" + from benchx.store import Store out = _dirty_run(repo, tmp, lambda base, head: [{"build": base, "source": repo["wt-base"]}, {"build": head, "source": repo["wt-head"]}], "listed") store = Store(tmp / "store.parquet") @@ -286,7 +303,9 @@ def test_head_and_history_show_the_dirty_warnings(repo, tmp, capsys): assert any(marker in point for point in points) and any(marker not in point for point in points) +@requires_pyarrow def test_tracked_series_and_history(revisions_run, tmp): + from benchx.store import Store store = Store(tmp / "store.parquet") store.ingest_paths([revisions_run]) series = store.series(workload=QUIET, quantity="wall-time") @@ -303,8 +322,10 @@ def test_tracked_series_and_history(revisions_run, tmp): assert changes[0] == [] and ["/benchmark/revision"] in changes +@requires_pyarrow def test_another_projection_on_request(revisions_run, tmp): """Schema §4.3: the same results under the revisions profile's projection, one series per tree.""" + from benchx.store import Store store = Store(tmp / "store.parquet") store.ingest_paths([revisions_run]) profile = compare.PROFILES["revisions"] @@ -394,7 +415,7 @@ def test_a_binary_that_cannot_list_its_benchmarks_is_a_refusal(repo, tmp, capsys with pytest.raises(runner.Refused, match=re.escape(message)): runner.run(path, tmp / "out") assert [p for p in (tmp / "out").glob("*.json") if not p.name.startswith("workorder-")] == [] - assert cli.main(["run", str(path), "--out", str(tmp / "out")]) == cli.EXIT_REFUSED + assert cli.main(["run", str(path), "--out", str(tmp / "out"), "--no-ingest"]) == cli.EXIT_REFUSED assert f"refused: {message}" in capsys.readouterr().err @@ -410,7 +431,7 @@ def test_a_filter_that_selects_no_case_is_a_refusal(repo, tmp, capsys): with pytest.raises(runner.Refused, match=re.escape(message)): runner.run(path, tmp / "out") assert [p for p in (tmp / "out").glob("*.json") if not p.name.startswith("workorder-")] == [] - assert cli.main(["run", str(path), "--out", str(tmp / "out")]) == cli.EXIT_REFUSED + assert cli.main(["run", str(path), "--out", str(tmp / "out"), "--no-ingest"]) == cli.EXIT_REFUSED assert f"refused: {message}" in capsys.readouterr().err @@ -502,17 +523,21 @@ def test_artifacts(repo, tmp): assert core.sha256_hex(native.read_bytes()) == artifacts["native-output"]["sha256"] +@requires_pyarrow def test_duplicate_keys_are_malformed(revisions_run, tmp): """S1: the one check only the strict loader makes.""" + from benchx.store import Store text = json.dumps(results(revisions_run)[0]) duplicated = text[:-1] + ', "ingest_key": "other"}' outcome = Store(tmp / "store.parquet").ingest([("dup.json", duplicated)]) assert outcome["rejected"][0]["code"] == "malformed" and "duplicate" in outcome["rejected"][0]["message"] +@requires_pyarrow def test_entities_auto_create(revisions_run, tmp): """S6: first sight creates the core rows of schema §4.1 (locked decision), derived from the one Parquet file that is the whole store.""" + from benchx.store import Store store = Store(tmp / "store.parquet") store.ingest_paths([revisions_run]) assert [p.name for p in tmp.glob("store*")] == ["store.parquet"] @@ -522,10 +547,12 @@ def test_entities_auto_create(revisions_run, tmp): assert len(entities["revision"]) == 2 +@requires_pyarrow def test_a_file_that_cannot_be_read_is_one_rejection_not_a_crash(adhoc, tmp, capsys): """Schema §5.4: a batch is N independent documents with N outcomes. A file that is not UTF-8, or is not there, is rejected as malformed, and the rest is ingested.""" from benchx import cli + from benchx.store import Store sweep = tmp / "sweep" sweep.mkdir() (sweep / "good.json").write_text(json.dumps(adhoc)) @@ -548,9 +575,11 @@ def test_a_file_that_cannot_be_read_is_one_rejection_not_a_crash(adhoc, tmp, cap assert "ingested 1, duplicate 0, rejected 3 (malformed)" in captured.out # into the CLI's own store +@requires_pyarrow def test_reads_files_from_an_older_schema(revisions_run, tmp): """A store written before a column existed takes new rows; old ones read the column as null.""" import pyarrow.parquet as pq + from benchx.store import Store files = sorted(p for p in revisions_run.glob("*.json") if not p.name.startswith("workorder-")) store = Store(tmp / "store.parquet") store.ingest_paths(files[:10]) @@ -561,8 +590,10 @@ def test_reads_files_from_an_older_schema(revisions_run, tmp): assert len(store.documents("rev-1")) == 48 +@requires_pyarrow def test_store_keeps_documents_exactly(revisions_run, tmp): """The document column is the canonical document, so nothing outside the file is needed.""" + from benchx.store import Store store = Store(tmp / "store.parquet") store.ingest_paths([revisions_run]) stored = {d["ingest_key"]: d for d in store.documents("rev-1")} @@ -570,8 +601,10 @@ def test_store_keeps_documents_exactly(revisions_run, tmp): assert core.canonical(stored[doc["ingest_key"]]) == core.canonical(doc) +@requires_pyarrow def test_head(revisions_run, tmp): """bx head: the latest rows, newest first.""" + from benchx.store import Store store = Store(tmp / "store.parquet") assert store.head() == (0, []) store.ingest_paths([revisions_run]) @@ -582,9 +615,11 @@ def test_head(revisions_run, tmp): assert json.loads(rows[0]["document"]) == load(ingested_last) +@requires_pyarrow def test_bx_run_delivers_to_the_local_store(repo, tmp, capsys): """bx run writes result files and ingests exactly those into the one local store.""" from benchx import cli + from benchx.store import Store build = make_build(tmp / "b", repo["wt-head"], cases()) for slot, name in enumerate(("o1", "o2")): (tmp / f"{name}.json").write_text(json.dumps(order(build, repo["wt-head"], "cli", slot=slot))) diff --git a/tests/test_refusals_not_crashes.py b/tests/test_refusals_not_crashes.py index c7339b4f..f9e57d06 100644 --- a/tests/test_refusals_not_crashes.py +++ b/tests/test_refusals_not_crashes.py @@ -3,13 +3,15 @@ Each is a refusal the user can act on: one line, naming what to change. """ +import json import subprocess import pytest from benchx import cli, session from benchx.adapters import asv, base, pyperf -from benchx.store import Store +from benchx.core import catalog +from benchx.results import MemoryStore MEASURE = ["A", "B", "--profile", "revisions", "--suite", "s", "--rounds", "1", "--adapter", "pyperf"] @@ -116,7 +118,7 @@ def stop(*args, **kwargs): with pytest.raises(session.SessionError, match="stop here"): session.compare_targets("A", "B", profile=profile, suite="s", rounds=1, adapter="pyperf", label=("env", "x", "y") if profile == "environments" else None, - store=Store(tmp / "s.parquet"), order_s=7) + store=MemoryStore(), order_s=7) assert seen == [7] @@ -144,3 +146,38 @@ def test_bx_bench_refuses_a_label_name_with_a_slash_as_it_records_it(capsys): "--label", "a/b=x"] assert cli.main(argv) == cli.EXIT_REFUSED assert f"bx bench: error: {expected.value}" in capsys.readouterr().err + + +# ---- missing jsonschema format checker in $BENCHX_SCHEMAS is refused, not a traceback + +def test_missing_format_checker_in_custom_schemas_is_refused(tmp_path, adhoc, monkeypatch, capsys): + """A custom $BENCHX_SCHEMAS declaring an unavailable format (e.g. `iri`) returns EXIT_REFUSED + with a single error line, not a traceback or EXIT_INVALID.""" + schemas = tmp_path / "schemas" + dest = schemas / "measurement-result" / "0.1.0" + dest.mkdir(parents=True) + schema = json.loads((catalog.SCHEMAS / "measurement-result" / "0.1.0" / "schema.json").read_text()) + schema["$defs"]["uri"]["format"] = "iri" + (dest / "schema.json").write_text(json.dumps(schema)) + + doc_dir = tmp_path / "results" + doc_dir.mkdir() + doc_path = doc_dir / "result.json" + doc_path.write_text(json.dumps(adhoc)) + + monkeypatch.setattr(catalog, "SCHEMAS", schemas) + catalog.clear_cache() + try: + assert cli.main(["validate", str(doc_path)]) == cli.EXIT_REFUSED + err = capsys.readouterr().err + assert "bx validate: error:" in err and "iri" in err and "Traceback" not in err + + assert cli.main([ + "compare", "--run", adhoc["provenance"]["run_key"], + "--profile", "revisions", "--baseline", adhoc["revision"]["key"][:8], + "--results", str(doc_dir), + ]) == cli.EXIT_REFUSED + err = capsys.readouterr().err + assert "bx compare: error:" in err and "iri" in err and "Traceback" not in err + finally: + catalog.clear_cache() diff --git a/tests/test_session.py b/tests/test_session.py index afda3165..cfda964b 100644 --- a/tests/test_session.py +++ b/tests/test_session.py @@ -6,8 +6,7 @@ import pytest from benchx import cli, runner, session -from benchx.store import Store -from conftest import NOISY, QUIET, cases, make_build +from conftest import NOISY, QUIET, cases, make_build, requires_pyarrow @pytest.fixture @@ -21,7 +20,9 @@ def verdicts(doc): return {u["workload"]["name"]: u["verdict"] for u in doc["units"]} +@requires_pyarrow def test_session_alternates_delivers_and_compares(repo, tmp, builds): + from benchx.store import Store store = Store(tmp / "store.parquet") doc = session.compare_targets(builds["baseline"], builds["contender"], profile="revisions", adapter="google-benchmark", suite="demo-bench", rounds=6, store=store, run_key="s1", out=tmp / "out") @@ -38,7 +39,9 @@ def test_session_alternates_delivers_and_compares(repo, tmp, builds): assert all(d["procedure"]["round"] == slot // 2 for slot, docs in by_slot.items() for d in docs) +@requires_pyarrow def test_session_environments(repo, tmp): + from benchx.store import Store plain = make_build(tmp / "plain", repo["wt-head"], cases()) hardened = make_build(tmp / "hardened", repo["wt-head"], cases(quiet=1.02e-3, noisy=2.06e-3), hardened=True) doc = session.compare_targets(f"{plain}:{repo['wt-head']}", f"{hardened}:{repo['wt-head']}", @@ -50,6 +53,7 @@ def test_session_environments(repo, tmp): def compare_dirty(builds, tmp, **extra): + from benchx.store import Store said = [] store = Store(tmp / "store.parquet") doc = session.compare_targets(builds["baseline"], builds["contender"], profile="revisions", @@ -58,6 +62,7 @@ def compare_dirty(builds, tmp, **extra): return doc, said +@requires_pyarrow def test_a_dirty_side_is_compared_and_labeled_local_only(repo, tmp, builds): """UC-03 §10: the comparison runs, says before measuring which side is not clean, and is local-only.""" (repo["wt-head"] / "bench.cpp").write_text("// uncommitted\n") @@ -71,6 +76,7 @@ def test_a_dirty_side_is_compared_and_labeled_local_only(repo, tmp, builds): assert {"contender: dirty-subject-tree", "contender: dirty-benchmark-tree"} <= caveats +@requires_pyarrow def test_both_dirty_sides_are_named(repo, tmp, builds): (repo["wt-base"] / "bench.cpp").write_text("// uncommitted, base\n") (repo["wt-head"] / "bench.cpp").write_text("// uncommitted, head\n") @@ -79,11 +85,13 @@ def test_both_dirty_sides_are_named(repo, tmp, builds): assert [m.split()[0] for m in said if "local-only" in m] == ["baseline", "contender"] +@requires_pyarrow def test_clean_sides_say_nothing_and_are_not_local_only(repo, tmp, builds): doc, said = compare_dirty(builds, tmp) assert doc["local_only"] is False and not [m for m in said if "local-only" in m] +@requires_pyarrow @pytest.mark.skipif(os.geteuid() == 0, reason="root reads any file") def test_a_side_with_no_tree_id_is_refused_before_anything_runs(repo, tmp, builds): """An unreadable untracked file leaves git without a working-tree id: the dirty state is `unknown`.""" @@ -106,20 +114,23 @@ def test_a_side_with_no_tree_id_is_refused_before_anything_runs(repo, tmp, build ({"profile": "history"}, "not in the implementation"), ]) def test_session_refusals(tmp, builds, kwargs, message): - args = {"adapter": "google-benchmark", "suite": "demo-bench", "rounds": 3, "store": Store(tmp / "store.parquet"), "out": tmp / "out", **kwargs} + from benchx.results import MemoryStore + args = {"adapter": "google-benchmark", "suite": "demo-bench", "rounds": 3, "store": MemoryStore(), "out": tmp / "out", **kwargs} with pytest.raises(session.SessionError, match=message): session.compare_targets(builds["baseline"], builds["contender"], **args) def test_a_refused_order_stops_the_session(tmp, builds, monkeypatch): + from benchx.results import MemoryStore def refuse(*_, **__): raise runner.Refused("deferred") monkeypatch.setattr(runner, "run", refuse) with pytest.raises(runner.Refused): session.compare_targets(builds["baseline"], builds["contender"], profile="revisions", - adapter="google-benchmark", suite="demo-bench", rounds=3, store=Store(tmp / "store.parquet"), out=tmp / "out") + adapter="google-benchmark", suite="demo-bench", rounds=3, store=MemoryStore(), out=tmp / "out") +@requires_pyarrow def test_measure_mode_equals_read_mode(repo, tmp, builds, capsys): common = ["--profile", "revisions"] assert cli.main(["compare", builds["baseline"], builds["contender"], *common, "--adapter", "google-benchmark", "--suite", "demo-bench", @@ -164,8 +175,8 @@ def test_mode_selection(argv, message, capsys): assert message in capsys.readouterr().err +@requires_pyarrow @pytest.mark.parametrize("argv, message", [ - # one target, and --baseline in measure mode: both are for variants only; no --vary is fine in either mode (["nowhere", "--profile", "variants", "--suite", "s", "--rounds", "3", "--adapter", "pyperf"], "working tree not found"), (["nowhere", "--profile", "variants", "--suite", "s", "--rounds", "3", "--adapter", "pyperf", diff --git a/tests/test_tables.py b/tests/test_tables.py index 84c6bcfa..05043d0a 100644 --- a/tests/test_tables.py +++ b/tests/test_tables.py @@ -3,7 +3,7 @@ import pytest from benchx import cli, compare, tables -from conftest import cases, make_build +from conftest import cases, make_build, requires_pyarrow from test_comparator import BASE, HEAD, result MEAN = "bench_a.Mean.time_mean_axis" @@ -205,6 +205,7 @@ def builds(repo, tmp): return {"baseline": f"{base}:{repo['wt-base']}", "contender": f"{head}:{repo['wt-head']}"} +@requires_pyarrow def test_tables_after_a_measure_mode_comparison_and_the_same_in_read_mode(repo, tmp, builds, capsys): common = ["--profile", "revisions"] assert cli.main(["compare", builds["baseline"], builds["contender"], *common, "--adapter", "google-benchmark", @@ -218,6 +219,7 @@ def test_tables_after_a_measure_mode_comparison_and_the_same_in_read_mode(repo, assert capsys.readouterr().out == measured +@requires_pyarrow def test_without_tables_the_output_is_what_it_was(repo, tmp, builds, capsys): cli.main(["compare", builds["baseline"], builds["contender"], "--profile", "revisions", "--adapter", "google-benchmark", "--suite", "demo-bench", "--rounds", "3", "--run-key", "t-2", "--out", str(tmp / "o")]) diff --git a/tests/test_target.py b/tests/test_target.py index 78ba474e..7ef6024b 100644 --- a/tests/test_target.py +++ b/tests/test_target.py @@ -9,8 +9,7 @@ from benchx import compare, config, core, runner, session, snapshot from benchx import target as target_mod -from benchx.store import Store -from conftest import SOURCE_URI, cases, make_build, order +from conftest import SOURCE_URI, cases, make_build, order, requires_pyarrow PROVIDER = """#!{python} import json, sys @@ -111,7 +110,9 @@ def test_provider_failure_and_missing_provider_are_errors(provider, tmp): config.load(tmp).require_provider("@plain") +@requires_pyarrow def test_declared_text_reaches_orders_results_and_the_comparison(repo, tmp, provider): + from benchx.store import Store store = Store(tmp / "store.parquet") doc = session.compare_targets("@plain", "@hardened", profile="environments", adapter="google-benchmark", suite="demo-bench", rounds=4, label=("build", "plain", "hardened"), project="demo", store=store, @@ -136,7 +137,9 @@ def test_declared_text_reaches_orders_results_and_the_comparison(repo, tmp, prov assert "hardened: activation: source env.sh" in compare.render(doc) +@requires_pyarrow def test_no_build_asks_the_provider_to_describe(repo, tmp, provider): + from benchx.store import Store session.compare_targets("@plain", "@hardened", profile="environments", adapter="google-benchmark", suite="demo-bench", rounds=3, label=("build", "plain", "hardened"), store=Store(tmp / "store.parquet"), run_key="t2", out=tmp / "out", build=False, provider=provider_of(".")) @@ -144,7 +147,9 @@ def test_no_build_asks_the_provider_to_describe(repo, tmp, provider): assert json.loads(result.read_text())["provenance"]["info"]["target_provider"]["version"] == "describe" +@requires_pyarrow def test_sidecar_route_needs_no_provider(repo, tmp): + from benchx.store import Store plain = make_build(tmp / "plain", repo["wt-head"], cases()) hardened = make_build(tmp / "hardened", repo["wt-head"], cases(quiet=1.02e-3), hardened=True) write_sidecar(hardened, repo["wt-head"], how_built="by hand") @@ -155,7 +160,9 @@ def test_sidecar_route_needs_no_provider(repo, tmp): assert doc["sides"] == {"baseline": {}, "contender": {"how_built": "by hand"}} +@requires_pyarrow def test_a_declared_build_is_refused_before_anything_runs(repo, tmp): + from benchx.store import Store build = make_build(tmp / "b", repo["wt-head"], cases()) write_sidecar(build, repo["wt-head"], build={"type": "Release"}) other = make_build(tmp / "o", repo["wt-head"], cases()) @@ -197,7 +204,8 @@ class _Stop(Exception): def _compare(tmp): - return session.compare_targets("a", "b", profile="revisions", suite="s", rounds=1, adapter="pyperf", store=Store(tmp / "s.parquet"), + from benchx.results import MemoryStore + return session.compare_targets("a", "b", profile="revisions", suite="s", rounds=1, adapter="pyperf", store=MemoryStore(), run_key="u", out=tmp / "out") diff --git a/tests/test_uc02.py b/tests/test_uc02.py index 7167d7c7..0c101939 100644 --- a/tests/test_uc02.py +++ b/tests/test_uc02.py @@ -12,6 +12,7 @@ import pytest pytest.importorskip("pyperf") +pytest.importorskip("pyarrow") from benchx import cli, core from conftest import git diff --git a/tests/test_uc04.py b/tests/test_uc04.py index 1bc0ab0f..894cbb33 100644 --- a/tests/test_uc04.py +++ b/tests/test_uc04.py @@ -13,6 +13,7 @@ import pytest pytest.importorskip("pyperf") +pytest.importorskip("pyarrow") from benchx import cli diff --git a/tests/test_views.py b/tests/test_views.py new file mode 100644 index 00000000..e90b559e --- /dev/null +++ b/tests/test_views.py @@ -0,0 +1,653 @@ +"""Derived tables, JSON view payloads, static site assembly, and `bx view` CLI.""" + +import datetime as dt +import json +import sys +import threading +import urllib.request +from pathlib import Path + +import pytest + +from benchx import cli, views +from benchx.views import inputs, marts as marts_mod, payloads, site +from conftest import NOISY, QUIET, cases, make_build, requires_pyarrow, run_rounds + +UTC = dt.timezone.utc +SHAS = [f"{i}" * 40 for i in range(1, 5)] + + +def _block_pyarrow(monkeypatch): + for name in list(sys.modules): + if name == "pyarrow" or name.startswith("pyarrow.") or name in ("benchx.store", "benchx.parquet"): + monkeypatch.delitem(sys.modules, name, raising=False) + monkeypatch.setitem(sys.modules, "pyarrow", None) + monkeypatch.setitem(sys.modules, "pyarrow.parquet", None) + + +def _point(sha, order, parent, case, estimate, status="success", quantity="wall-time", direction="lower-is-better"): + return { + "sha": sha, + "commit_order": order, + "committer_date": dt.datetime(2026, 1, order, tzinfo=UTC), + "parent": parent, + "node": "n1", + "suite": "s", + "case": case, + "params": "", + "quantity": quantity, + "unit": "s", + "direction": direction, + "status": status, + "reason": "" if status == "success" else "harness.no-output", + "estimate": estimate, + "mean": estimate, + "min": estimate, + "max": estimate, + "n": 1 if estimate is not None else 0, + "started_at": dt.datetime(2026, 1, order, tzinfo=UTC), + "run_key": f"main/{sha}/n1", + "ingest_key": f"{sha}:{case}", + } + + +def _by(rows, **keys): + out = [r for r in rows if all(r[k] == v for k, v in keys.items())] + assert len(out) == 1, (keys, out) + return out[0] + + +# --- Marts arithmetic (pure Python, no PyArrow required) ------------------------------- + +@pytest.fixture +def synthetic_marts(): + pts = [ + _point("c1", 1, "c0", "steady", 1.00), + _point("c2", 2, "c1", "steady", 1.01), + _point("c3", 3, "c2", "steady", 1.10), + _point("c1", 1, "c0", "quiet", 2.00), + _point("c3", 3, "c2", "quiet", 2.02), + _point("c3", 3, "c2", "broken", None, status="error"), + ] + changes = marts_mod.build_changes(pts) + latest = marts_mod.build_latest(pts) + movers = marts_mod.build_movers(changes) + return {"changes": changes, "latest": latest, "movers": movers} + + +def test_changes_follow_the_series_and_record_gaps(synthetic_marts): + ch = synthetic_marts["changes"] + assert len(ch) == 3 # steady: c2, c3; quiet: c3; broken has no successful point + c3 = _by(ch, case="steady", sha="c3") + assert c3["prev_sha"] == "c2" and c3["prev_is_parent"] is True and c3["gap"] == 1 + assert c3["rel_change"] == pytest.approx(1.10 / 1.01 - 1) + q3 = _by(ch, case="quiet", sha="c3") + assert q3["prev_sha"] == "c1" and q3["prev_is_parent"] is False and q3["gap"] == 2 + assert q3["rel_change"] == pytest.approx(0.01) + + +def test_latest_summarizes_each_series(synthetic_marts): + lt = synthetic_marts["latest"] + assert {r["case"] for r in lt} == {"steady", "quiet"} + s = _by(lt, case="steady") + assert s["sha"] == "c3" and s["points"] == 3 and s["first_order"] == 1 + assert s["estimate"] == pytest.approx(1.10) + assert s["vs_prev"] == pytest.approx(1.10 / 1.01 - 1) + assert s["vs_10"] is None and s["vs_100"] is None + assert s["noise"] == pytest.approx((abs(1.10 / 1.01 - 1) + 0.01) / 2) + q = _by(lt, case="quiet") + assert q["noise"] is None + assert s["noise_rank"] == 0.0 and q["noise_rank"] == 1.0 + + +def test_movers_rank_changes_per_commit(synthetic_marts): + mv = synthetic_marts["movers"] + c3 = [r for r in mv if r["sha"] == "c3"] + assert [r["case"] for r in c3] == ["steady", "quiet"] + assert all(r["rel_change"] is not None for r in mv) + + +def test_build_commits_orders_by_committer_date(): + runs = [ + {"sha": "b" * 40, "parents": ["a" * 40], "committer_date": inputs.parse_time("2026-02-01T10:00:00+00:00"), + "author_date": inputs.parse_time("2026-02-01T10:00:00+00:00"), "subject": "second", "node": "n1", + "state": "finished", "results": 5}, + {"sha": "a" * 40, "parents": ["9" * 40], "committer_date": inputs.parse_time("2026-01-01T10:00:00+01:00"), + "author_date": inputs.parse_time("2026-01-01T10:00:00+01:00"), "subject": "first", "node": "n1", + "state": "finished", "results": 5}, + ] + commits = marts_mod.build_commits(runs) + order = {r["sha"]: r["commit_order"] for r in commits} + assert order["a" * 40] == 1 and order["b" * 40] == 2 + assert _by(commits, sha="a" * 40)["parent"] == "9" * 40 + + +# --- View payloads (pure Python, no PyArrow required) ---------------------------------- + +def _build_synthetic_views(tmp_path: Path, raw_points: list[dict], n_commits: int = 4) -> Path: + runs = [ + { + "sha": SHAS[i - 1], + "parents": [SHAS[i - 2] if i > 1 else "0" * 40], + "committer_date": dt.datetime(2026, 1, i, tzinfo=UTC), + "author_date": dt.datetime(2026, 1, i, tzinfo=UTC), + "subject": f"commit {i}", + "branch": "main", + "node": "n1", + "state": "finished", + "reason": None, + "started_at": dt.datetime(2026, 1, i, tzinfo=UTC), + "finished_at": dt.datetime(2026, 1, i, tzinfo=UTC), + "suites_ok": 1, + "suites_failed": 0, + "results": 3, + "ingest_problems": 0, + "log": "", + } + for i in range(1, n_commits + 1) + ] + marts = marts_mod.build_marts(runs, raw_points, source="runs") + data = tmp_path / "data" + data.mkdir() + payloads.build_views(marts, data) + return data / "views" + + +def _load(path: Path): + return json.loads(path.read_text(encoding="utf-8")) + + +@pytest.fixture +def view_out(tmp_path): + pts = [ + _point(SHAS[0], 1, "0" * 40, "flaky", None, "error"), + _point(SHAS[1], 2, SHAS[0], "flaky", 1.0), + _point(SHAS[2], 3, SHAS[1], "flaky", None, "error"), + _point(SHAS[3], 4, SHAS[2], "flaky", 1.1), + *[_point(SHAS[i - 1], i, SHAS[i - 2] if i > 1 else "0" * 40, "broken", None, "error") for i in range(1, 5)], + *[_point(SHAS[i - 1], i, SHAS[i - 2] if i > 1 else "0" * 40, "fast", v, + quantity="items-per-second", direction="higher-is-better") + for i, v in [(1, 100), (2, 101), (3, 100), (4, 80)]], + ] + return _build_synthetic_views(tmp_path, pts) + + +def test_history_keeps_each_estimate_on_its_commit(view_out): + flaky = _load(view_out / "history" / "s.json")["series"]["flaky||wall-time|n1"] + assert flaky["orders"] == [1, 2, 3, 4] + assert flaky["estimates"] == [None, 1.0, None, 1.1] + assert flaky["statuses"] == ["error", "success", "error", "success"] + spark = next(s for s in _load(view_out / "sparks.json")["series"] if s["case"] == "flaky") + assert spark["orders"] == [2, 4] and spark["estimates"] == [1.0, 1.1] + + +def test_failures_are_listed_not_hidden(view_out): + series = {s["case"]: s for s in _load(view_out / "series.json")} + assert series["broken"]["estimate"] is None and series["broken"]["status"] == "error" and series["broken"]["points"] == 0 + assert series["flaky"]["status"] == "success" + landing = _load(view_out / "landing.json") + assert [f["case"] for f in landing["failures"]] == ["broken"] + assert landing["latest_runs"][0]["failed_results"] == 1 + newest = _load(view_out / "commits" / f"{SHAS[3]}.json") + assert [f["case"] for f in newest["failures"]] == ["broken"] + + +def test_counts_follow_the_direction(view_out): + newest = _load(view_out / "landing.json")["latest_runs"][0] + assert newest["regressed"] == 2 and newest["improved"] == 0 + assert {m["case"] for m in _load(view_out / "landing.json")["movers"]["regressed"]} == {"flaky", "fast"} + + +def test_no_results_still_builds(tmp_path): + out = _build_synthetic_views(tmp_path, [], n_commits=2) + landing = _load(out / "landing.json") + assert landing["stats"]["results"] == 0 and len(landing["recent_commits"]) == 2 + assert _load(out / "series.json") == [] and _load(out / "sparks.json") == {"axis": [], "series": []} + assert _load(out / "commits" / f"{SHAS[1]}.json")["changes"] == [] + + +def test_last_executed_is_by_finish_time_not_commit_order(): + commits = [ + {"sha": "new", "node": "n1", "finished_at": "2026-01-02T10:00:00+00:00", "started_at": None}, + {"sha": "old", "node": "n1", "finished_at": "2026-01-03T09:00:00+00:00", "started_at": None}, + {"sha": "new", "node": "n2", "finished_at": None, "started_at": "2026-01-01T08:00:00+00:00"}, + ] + assert payloads.newest_commits(commits)["n1"]["sha"] == "new" + last = payloads.last_executed(commits) + assert last["n1"]["sha"] == "old" and last["n2"]["sha"] == "new" + + +# --- End-to-end JSON results mode (Arrow-free) ----------------------------------------- + +def test_view_from_json_results_without_pyarrow(repo, tmp, monkeypatch, capsys): + _block_pyarrow(monkeypatch) + base = make_build(tmp / "build-base", repo["wt-base"], cases()) + head = make_build(tmp / "build-head", repo["wt-head"], cases(quiet=1.10e-3, noisy=2.00e-3)) + + out_base = tmp / "res-base" + out_head = tmp / "res-head" + out_dirty = tmp / "res-dirty" + for target, out_dir, rkey in ( + (f"{base}:{repo['wt-base']}", out_base, "r-base"), + (f"{head}:{repo['wt-head']}", out_head, "r-head"), + ): + assert cli.main([ + "bench", target, "--adapter", "google-benchmark", "--suite", "demo-bench", + "--run-key", rkey, "--out", str(out_dir), "--no-ingest", + ]) == cli.EXIT_OK + + # Make an uncommitted edit on wt-head and benchmark it too. + (repo["wt-head"] / "bench.cpp").write_text("// dirty edit\n") + assert cli.main([ + "bench", f"{head}:{repo['wt-head']}", "--adapter", "google-benchmark", "--suite", "demo-bench", + "--run-key", "r-dirty", "--out", str(out_dirty), "--no-ingest", + ]) == cli.EXIT_OK + capsys.readouterr() + + site_dir = tmp / "site" + assert cli.main([ + "view", + "--results", str(out_base), str(out_head), str(out_dirty), + "--repo", str(repo["src"]), + "--title", "Demo Local", + "--stamp", "v1", + "--out", str(site_dir), + ]) == cli.EXIT_OK + stdout = capsys.readouterr().out + assert "3 commit(s)" in stdout + + # Static assets stamped and present, no Parquet marts written without PyArrow. + html = (site_dir / "index.html").read_text(encoding="utf-8") + assert 'href="style.css?v=v1"' in html and 'src="app.js?v=v1"' in html + for asset in ("app.js", "style.css", "favicon.svg", "logo.svg"): + assert (site_dir / asset).is_file() + + manifest = _load(site_dir / "data" / "manifest.json") + assert manifest["source"] == "results" + assert manifest["tables"] == {} + assert manifest["counts"]["distinct_commits"] == 3 + assert manifest["counts"]["points"] == 6 + + project = _load(site_dir / "data" / "project.json") + assert project["title"] == "Demo Local" + + landing = _load(site_dir / "data" / "views" / "landing.json") + subjects = {c["sha"]: c["subject"] for c in landing["recent_commits"]} + shorts = {c["sha"]: c["short"] for c in landing["recent_commits"]} + assert subjects[repo["base"]] == "base" + assert subjects[repo["head"]] == "head" + dirty_shas = [sha for sha in subjects if "-dirty-" in sha] + assert len(dirty_shas) == 1 + assert "dirty working tree" in subjects[dirty_shas[0]] + assert "+dirty" in shorts[dirty_shas[0]] and shorts[dirty_shas[0]] != shorts[repo["head"]] + + # Head's parent is base, verified from git metadata (non-vacuous even on equal 1s timestamps). + head_page = _load(site_dir / "data" / "views" / "commits" / f"{repo['head']}.json") + assert head_page["parent"] == repo["base"] + assert head_page["changes"] and all(ch["prev_is_parent"] is True for ch in head_page["changes"]) + + +def test_view_pyarrow_errors_are_actionable(tmp, monkeypatch, capsys): + _block_pyarrow(monkeypatch) + assert cli.main(["view", "--out", str(tmp / "site")]) == cli.EXIT_REFUSED + err = capsys.readouterr().err + assert "benchx[parquet]" in err and "--results" in err + + res = tmp / "empty" + res.mkdir() + assert cli.main(["view", "--results", str(res), "--marts", "--out", str(tmp / "site2")]) == cli.EXIT_REFUSED + err = capsys.readouterr().err + assert "benchx[parquet]" in err + + +# --- End-to-end Parquet store & CI runs/ modes ----------------------------------------- + +@requires_pyarrow +def test_view_from_local_store_collapses_rounds_and_writes_marts(repo, tmp, capsys): + import pyarrow.parquet as pq + from benchx.store import Store + + base = make_build(tmp / "b-base", repo["wt-base"], cases()) + head = make_build(tmp / "b-head", repo["wt-head"], cases(quiet=1.10e-3, noisy=2.00e-3)) + out = run_rounds( + tmp, + [{"build": base, "source": repo["wt-base"]}, {"build": head, "source": repo["wt-head"]}], + rounds=3, + run_key="cmp-1", + out=tmp / "results-cmp", + ) + store = Store(tmp / "store.parquet") + assert store.ingest_paths([out])["rejected"] == [] + + site_dir = tmp / "site" + assert cli.main([ + "view", "--store", str(tmp / "store.parquet"), + "--repo", str(repo["src"]), "--out", str(site_dir), + ]) == cli.EXIT_OK + + manifest = _load(site_dir / "data" / "manifest.json") + assert manifest["source"] == "store" + assert set(manifest["tables"]) == {"commits", "points", "changes", "latest", "movers"} + # 2 commits x 2 cases collapsed across 3 rounds (5 reps each -> n=15 per commit). + assert manifest["counts"]["points"] == 4 + points = pq.read_table(site_dir / "data" / "points.parquet").to_pylist() + assert len(points) == 4 and all(p["n"] == 15 for p in points) + assert {p["case"] for p in points} == {QUIET, NOISY} + + +@requires_pyarrow +def test_view_from_runs_dir_with_logs_and_failed_build(repo, tmp, capsys): + import pyarrow.parquet as pq + from benchx.store import Store + + runs_dir = tmp / "runs" + base = make_build(tmp / "b-base", repo["wt-base"], cases()) + head = make_build(tmp / "b-head", repo["wt-head"], cases(quiet=1.12e-3, noisy=2.00e-3)) + + for sha, parent, date, build_dir, wt in ( + (repo["base"], "0" * 40, "2026-01-01T10:00:00+00:00", base, repo["wt-base"]), + (repo["head"], repo["base"], "2026-01-02T10:00:00+00:00", head, repo["wt-head"]), + ): + res_out = tmp / f"raw-{sha[:8]}" + assert cli.main([ + "bench", f"{build_dir}:{wt}", "--adapter", "google-benchmark", "--suite", "demo-bench", + "--run-key", f"main/{sha}/m5", "--out", str(res_out), "--no-ingest", + ]) == cli.EXIT_OK + node_dir = runs_dir / sha / "m5" + Store(node_dir / "results.parquet").ingest_paths([res_out]) + views.write_run( + node_dir, + {"sha": sha, "parents": [parent], "committer_date": date, "author_date": date, "subject": f"msg {sha[:6]}"}, + {"state": "finished", "node": "m5", "suites_ok": 1, "suites_failed": 0, "suites": {"demo-bench": {"results": 2}}}, + log=f"log for {sha[:6]}\n", + ) + + # A third commit whose build failed (run.parquet only, no results.parquet). + failed_sha = "f" * 40 + views.write_run( + runs_dir / failed_sha / "m5", + {"sha": failed_sha, "parents": [repo["head"]], "committer_date": "2026-01-03T10:00:00+00:00", "subject": "broken build"}, + {"state": "failed", "node": "m5"}, + log="cmake failed\n", + ingest_problems=1, + ) + + project_file = tmp / "project.json" + project_file.write_text(json.dumps({ + "name": "demo", + "title": "Demo CI Benchmarks", + "repository": "https://github.com/example/benchx-demo", + "upstream": {"url": "https://github.com/example/demo.git", "branch": "main", + "commit_url": "https://github.com/example/demo/commit/{sha}"}, + })) + capsys.readouterr() + + site_dir = tmp / "site" + assert cli.main([ + "view", "--runs", str(runs_dir), "--project-file", str(project_file), "--out", str(site_dir), + ]) == cli.EXIT_OK + + manifest = _load(site_dir / "data" / "manifest.json") + assert manifest["source"] == "runs" + assert manifest["counts"]["distinct_commits"] == 3 + assert manifest["counts"]["ingest_problems"] == 1 + assert (site_dir / "data" / "logs" / failed_sha / "m5.txt").read_text(encoding="utf-8") == "cmake failed\n" + + failed_page = _load(site_dir / "data" / "views" / "commits" / f"{failed_sha}.json") + assert failed_page["nodes"][0]["state"] == "failed" and failed_page["nodes"][0]["has_log"] is True + assert pq.read_table(site_dir / "data" / "commits.parquet").num_rows == 3 + + +# --- Local HTTP serving & default temp-dir behavior ------------------------------------ + +def test_serve_and_default_temp_dir(tmp, monkeypatch): + res_dir = tmp / "empty-res" + res_dir.mkdir() + site_dir = tmp / "built" + views.build_site(site_dir, results_paths=[res_dir], write_marts=False) + + server, url = site.create_server(site_dir, port=0) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + with urllib.request.urlopen(url) as resp: + assert resp.status == 200 + assert b"Benchmarks" in resp.read() + with urllib.request.urlopen(url + "data/manifest.json") as resp: + assert resp.status == 200 + assert json.loads(resp.read())["source"] == "results" + finally: + server.shutdown() + server.server_close() + thread.join(timeout=2) + + # Without --out, `bx view` builds into a temporary directory and serves it. + served_dirs = [] + + def fake_serve(directory, **kwargs): + assert (Path(directory) / "index.html").is_file() + assert (Path(directory) / "data" / "views" / "landing.json").is_file() + served_dirs.append(Path(directory)) + + monkeypatch.setattr(views, "serve_site", fake_serve) + assert cli.main(["view", "--results", str(res_dir)]) == cli.EXIT_OK + assert len(served_dirs) == 1 + assert not served_dirs[0].exists() # temp directory cleaned up after serve exits + + +def test_parametrized_workload_and_rejected_results(adhoc, tmp, monkeypatch, capsys): + _block_pyarrow(monkeypatch) + res_dir = tmp / "param-res" + res_dir.mkdir() + doc = json.loads(json.dumps(adhoc)) + doc["coordinates"]["workload"]["parameters"] = {"b": 2, "a": 1} + (res_dir / "result-1.json").write_text(json.dumps(doc), encoding="utf-8") + + site_dir = tmp / "param-site" + assert cli.main(["view", "--results", str(res_dir), "--out", str(site_dir)]) == cli.EXIT_OK + series = _load(site_dir / "data" / "views" / "series.json") + assert len(series) == 1 + assert series[0]["params"] == '{"a":1,"b":2}' + + # Malformed JSON file is reported, still prints summary, and exits EXIT_INVALID. + (res_dir / "broken.json").write_text("{not valid", encoding="utf-8") + capsys.readouterr() + bad_site = tmp / "bad-site" + assert cli.main(["view", "--results", str(res_dir), "--out", str(bad_site)]) == cli.EXIT_INVALID + captured = capsys.readouterr() + assert "rejected broken.json: malformed" in captured.err + assert str(bad_site) in captured.out + + # Without --out, still serves the valid results (with ingest_problems banner) and returns EXIT_INVALID. + served = [] + monkeypatch.setattr(views, "serve_site", lambda directory, **kwargs: served.append(Path(directory))) + assert cli.main(["view", "--results", str(res_dir)]) == cli.EXIT_INVALID + assert len(served) == 1 + + +def test_build_commits_breaks_equal_timestamp_ties_by_ancestry(): + same_ts = dt.datetime(2026, 3, 1, 12, 0, 0, tzinfo=UTC) + parent_sha, child_sha = "f" * 40, "0" * 40 # child sorts before parent lexicographically + runs = [ + {"sha": child_sha, "parents": [parent_sha], "committer_date": same_ts, "author_date": same_ts, + "subject": "child", "node": "n1", "state": "finished", "results": 1}, + {"sha": parent_sha, "parents": ["a" * 40], "committer_date": same_ts, "author_date": same_ts, + "subject": "parent", "node": "n1", "state": "finished", "results": 1}, + ] + commits = marts_mod.build_commits(runs) + order = {r["sha"]: r["commit_order"] for r in commits} + assert order[parent_sha] == 1 and order[child_sha] == 2 + + +def test_summary_only_estimate_and_partial_and_skipped_statuses(adhoc, tmp, monkeypatch): + _block_pyarrow(monkeypatch) + res_dir = tmp / "status-res" + res_dir.mkdir() + + # 1. Summary-only success result (no observations, schema estimate summary). + summary_only = json.loads(json.dumps(adhoc)) + summary_only["ingest_key"] = "sum-1" + summary_only["attempt_key"] = "att-sum-1" + summary_only["coordinates"]["workload"]["name"] = "case_summary" + summary_only["measurement"].pop("observations", None) + summary_only["measurement"]["summaries"] = [ + { + "type": "estimate", + "estimator": {"name": "median", "method_version": "benchx/median/v1", "input_level": "observations"}, + "value": 0.123, + "source": "producer", + } + ] + (res_dir / "summary.json").write_text(json.dumps(summary_only), encoding="utf-8") + + # 2. Partial result with observations: contributes an estimate and is not a failure. + partial_doc = json.loads(json.dumps(adhoc)) + partial_doc["ingest_key"] = "part-1" + partial_doc["attempt_key"] = "att-part-1" + partial_doc["coordinates"]["workload"]["name"] = "case_partial" + partial_doc["measurement"]["status"] = "partial" + partial_doc["measurement"]["reason"] = "budget-exceeded" + (res_dir / "partial.json").write_text(json.dumps(partial_doc), encoding="utf-8") + + # 3. Skipped result: listed in series.json, not counted as a failure. + skipped_doc = json.loads(json.dumps(adhoc)) + skipped_doc["ingest_key"] = "skip-1" + skipped_doc["attempt_key"] = "att-skip-1" + skipped_doc["coordinates"]["workload"]["name"] = "case_skipped" + skipped_doc["measurement"]["status"] = "skipped" + skipped_doc["measurement"]["reason"] = "unsupported" + skipped_doc["measurement"].pop("observations", None) + (res_dir / "skipped.json").write_text(json.dumps(skipped_doc), encoding="utf-8") + + site_dir = tmp / "status-site" + assert cli.main(["view", "--results", str(res_dir), "--out", str(site_dir)]) == cli.EXIT_OK + + series = {s["case"]: s for s in _load(site_dir / "data" / "views" / "series.json")} + assert series["case_summary"]["estimate"] == pytest.approx(0.123) and series["case_summary"]["points"] == 1 + assert series["case_partial"]["estimate"] is not None and series["case_partial"]["points"] == 1 + assert series["case_skipped"]["status"] == "skipped" and series["case_skipped"]["estimate"] is None + + landing = _load(site_dir / "data" / "views" / "landing.json") + assert landing["failures"] == [] + assert landing["latest_runs"][0]["failed_results"] == 0 + + +@pytest.mark.parametrize("flag, uri", [ + ("--results", "s3://bucket/results"), + ("--store", "s3://bucket/store.parquet"), + ("--runs", "s3://bucket/runs"), +]) +def test_remote_uri_rejected_clearly(tmp, monkeypatch, capsys, flag, uri): + _block_pyarrow(monkeypatch) + out = tmp / "remote-site" + assert cli.main(["view", flag, uri, "--out", str(out)]) == cli.EXIT_REFUSED + err = capsys.readouterr().err + assert "remote URI" in err and "s3://" in err + assert not out.exists() + + +@requires_pyarrow +def test_path_traversal_and_orphan_results_in_runs_dir(repo, tmp, capsys): + from benchx.store import Store + + runs_dir = tmp / "runs" + base = make_build(tmp / "b-base", repo["wt-base"], cases()) + res_out = tmp / "raw-base" + assert cli.main([ + "bench", f"{base}:{repo['wt-base']}", "--adapter", "google-benchmark", "--suite", "demo-bench", + "--run-key", f"main/{repo['base']}/m5", "--out", str(res_out), "--no-ingest", + ]) == cli.EXIT_OK + # Orphan results.parquet without a matching run.parquet. + Store(runs_dir / repo["base"] / "m5" / "results.parquet").ingest_paths([res_out]) + + # Malicious path-traversal node name in a second run's run.parquet. + evil_sha = "e" * 40 + views.write_run( + runs_dir / evil_sha / "safe-dir", + {"sha": evil_sha, "parents": [], "committer_date": "2026-01-01T00:00:00+00:00", "subject": "evil"}, + {"state": "failed", "node": "../../../../escaped"}, + log="must stay inside out\n", + ) + capsys.readouterr() + + site_dir = tmp / "site" + assert cli.main(["view", "--runs", str(runs_dir), "--out", str(site_dir)]) == cli.EXIT_OK + + # Traversal log stayed inside site_dir/data/logs and did not escape. + assert not (tmp / "escaped.txt").exists() + logs = list((site_dir / "data" / "logs").rglob("*.txt")) + assert len(logs) == 1 and logs[0].resolve().is_relative_to((site_dir / "data" / "logs").resolve()) + + # Orphan results got a synthesized commit row (commit_order > 0) and a commit page. + landing = _load(site_dir / "data" / "views" / "landing.json") + by_sha = {c["sha"]: c for c in landing["recent_commits"]} + assert by_sha[repo["base"]]["commit_order"] > 0 + assert (site_dir / "data" / "views" / "commits" / f"{repo['base']}.json").is_file() + + +def test_validation_errors_do_not_create_out_and_rebuild_prunes_stale_files(tmp, monkeypatch): + _block_pyarrow(monkeypatch) + res_dir = tmp / "res" + res_dir.mkdir() + + # Invalid/missing --project-file fails before creating --out. + bad_out = tmp / "bad-proj-out" + assert cli.main([ + "view", "--results", str(res_dir), "--project-file", str(tmp / "missing.json"), "--out", str(bad_out), + ]) == cli.EXIT_REFUSED + assert not bad_out.exists() + + # --marts without pyarrow fails before creating --out. + marts_out = tmp / "marts-out" + assert cli.main(["view", "--results", str(res_dir), "--marts", "--out", str(marts_out)]) == cli.EXIT_REFUSED + assert not marts_out.exists() + + # Rebuilding with --no-marts prunes stale .parquet marts and stale logs. + site_dir = tmp / "rebuild-site" + stale_parquet = site_dir / "data" / "points.parquet" + stale_log = site_dir / "data" / "logs" / "oldsha" / "n1.txt" + stale_parquet.parent.mkdir(parents=True) + stale_parquet.write_bytes(b"stale") + stale_log.parent.mkdir(parents=True) + stale_log.write_text("old log", encoding="utf-8") + + assert cli.main(["view", "--results", str(res_dir), "--no-marts", "--out", str(site_dir)]) == cli.EXIT_OK + assert not stale_parquet.exists() + assert not stale_log.exists() + + +def test_safe_path_segment_strips_windows_unsafe_and_traversal_chars(): + assert inputs.safe_path_segment("../../../../escaped") == "________escaped" + assert inputs.safe_path_segment("tag:v1*2?3<4>5|6") == "tag_v1_2_3_4_5_6" + assert inputs.safe_path_segment(".") == "_" + assert inputs.safe_path_segment("") == "_" + assert inputs.safe_path_segment("mod.py") == "mod.py" + + +@requires_pyarrow +def test_runs_dir_collapses_duplicate_attempts_without_self_changes(repo, tmp): + from benchx.store import Store + + runs_dir = tmp / "runs" + base = make_build(tmp / "b-base", repo["wt-base"], cases()) + out = run_rounds( + tmp, + [{"build": base, "source": repo["wt-base"]}], + rounds=2, + run_key="dup-run", + out=tmp / "raw-dup", + ) + store = Store(runs_dir / repo["base"] / "m5" / "results.parquet") + store.ingest_paths([out]) + node = inputs.read_store(store.path)[0][0]["node"] + views.write_run( + runs_dir / repo["base"] / "m5", + {"sha": repo["base"], "parents": [], "committer_date": "2026-01-01T10:00:00+00:00", "subject": "base"}, + {"state": "finished", "node": node}, + ) + + site_dir = tmp / "site" + assert cli.main(["view", "--runs", str(runs_dir), "--out", str(site_dir)]) == cli.EXIT_OK + manifest = _load(site_dir / "data" / "manifest.json") + assert manifest["counts"]["points"] == 2 # 2 cases collapsed across 2 rounds + assert manifest["counts"]["changes"] == 0 # no self-comparison on the same commit diff --git a/uv.lock b/uv.lock index 65762fc3..59c3ed3c 100644 --- a/uv.lock +++ b/uv.lock @@ -2,19 +2,6 @@ version = 1 revision = 3 requires-python = ">=3.11" -[[package]] -name = "arrow" -version = "1.4.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "python-dateutil" }, - { name = "tzdata" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b9/33/032cdc44182491aa708d06a68b62434140d8c50820a087fac7af37703357/arrow-1.4.0.tar.gz", hash = "sha256:ed0cc050e98001b8779e84d461b0098c4ac597e88704a655582b21d116e526d7", size = 152931, upload-time = "2025-10-18T17:46:46.761Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ed/c9/d7977eaacb9df673210491da99e6a247e93df98c715fc43fd136ce1d3d33/arrow-1.4.0-py3-none-any.whl", hash = "sha256:749f0769958ebdc79c173ff0b0670d59051a535fa26e8eba02953dc19eb43205", size = 68797, upload-time = "2025-10-18T17:46:45.663Z" }, -] - [[package]] name = "asv-runner" version = "0.3.1" @@ -37,14 +24,17 @@ wheels = [ name = "benchx" source = { editable = "." } dependencies = [ - { name = "jsonpointer" }, - { name = "jsonschema", extra = ["format-nongpl"] }, - { name = "pyarrow" }, + { name = "jsonschema" }, { name = "referencing" }, + { name = "rfc3339-validator" }, + { name = "rfc3986-validator" }, { name = "rfc8785" }, ] [package.optional-dependencies] +parquet = [ + { name = "pyarrow" }, +] test = [ { name = "asv-runner" }, { name = "pyperf" }, @@ -54,15 +44,16 @@ test = [ [package.metadata] requires-dist = [ { name = "asv-runner", marker = "extra == 'test'" }, - { name = "jsonpointer", specifier = ">=2.0" }, - { name = "jsonschema", extras = ["format-nongpl"], specifier = ">=4.18" }, - { name = "pyarrow", specifier = ">=19" }, + { name = "jsonschema", specifier = ">=4.18" }, + { name = "pyarrow", marker = "extra == 'parquet'", specifier = ">=19" }, { name = "pyperf", marker = "extra == 'test'" }, { name = "pytest", marker = "extra == 'test'" }, { name = "referencing" }, + { name = "rfc3339-validator" }, + { name = "rfc3986-validator", specifier = ">0.1.0" }, { name = "rfc8785" }, ] -provides-extras = ["test"] +provides-extras = ["parquet", "test"] [[package]] name = "colorama" @@ -73,24 +64,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, ] -[[package]] -name = "fqdn" -version = "1.6.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/02/01/248d44fbda9e78fa1a7b5a475194bef4c4a3e285db286419f3bb42f2efdf/fqdn-1.6.0.tar.gz", hash = "sha256:e39bf62e2a9481aa2cb780554fdfd944dd8ee5f230eb831684d539cbafed8de2", size = 130737, upload-time = "2026-09-28T05:43:35.482Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/15/d5/a8f9149b97cced9e1f4bff92ff0b6206120e2c853ae774d7020d6ecd8fa8/fqdn-1.6.0-py3-none-any.whl", hash = "sha256:6c793c312ccfa29981e1135b3ef5e1277ebe639731a6b0f7350d90e96e63f0b4", size = 12569, upload-time = "2026-09-28T05:43:33.951Z" }, -] - -[[package]] -name = "idna" -version = "3.20" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f5/08/8eea9d4b8302028f3abb2c0813953f7aec26d33b7a8960ed760e65ff29fa/idna-3.20.tar.gz", hash = "sha256:a7db850025b95ded1eae8a46181a1a6c56c92c96f0e2b005d9ff8dc0210cab44", size = 216463, upload-time = "2026-09-17T14:11:04.752Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/58/a2/bb081bab032533a855d44de1d56f8e8426114ff1ba5d1f07a438a0a654f8/idna-3.20-py3-none-any.whl", hash = "sha256:ab7ae7122974553370f0bdb919e1a960b2cd1bc1ef0276416d896db81c14582c", size = 69583, upload-time = "2026-09-17T14:11:03.168Z" }, -] - [[package]] name = "iniconfig" version = "2.3.0" @@ -100,27 +73,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, ] -[[package]] -name = "isoduration" -version = "20.11.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "arrow" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/7c/1a/3c8edc664e06e6bd06cce40c6b22da5f1429aa4224d0c590f3be21c91ead/isoduration-20.11.0.tar.gz", hash = "sha256:ac2f9015137935279eac671f94f89eb00584f940f5dc49462a0c4ee692ba1bd9", size = 11649, upload-time = "2020-11-01T11:00:00.312Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7b/55/e5326141505c5d5e34c5e0935d2908a74e4561eca44108fbfb9c13d2911a/isoduration-20.11.0-py3-none-any.whl", hash = "sha256:b2904c2a4228c3d44f409c8ae8e2370eb21a26f7ac2ec5446df141dde3452042", size = 11321, upload-time = "2020-11-01T10:59:58.02Z" }, -] - -[[package]] -name = "jsonpointer" -version = "3.2.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/33/a2/c92f0a7ed439c490d2c8ad712fdb074c311afa4f77870987826a3b1483ba/jsonpointer-3.2.1.tar.gz", hash = "sha256:47c846513b3a4ec46eecef1105207fba075e2a3659048e362bd7daff0fc33342", size = 11309, upload-time = "2026-10-10T21:11:06.988Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fa/29/accef8eea16670b88f3a23102c35dee62c6f159db30d9628908043b3e21b/jsonpointer-3.2.1-py3-none-any.whl", hash = "sha256:b19ee68644e9ffb51440448d8f7811af2b7406eea1db90603e93f5849323119a", size = 8332, upload-time = "2026-10-10T21:11:05.648Z" }, -] - [[package]] name = "jsonschema" version = "4.26.0" @@ -136,19 +88,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/69/90/f63fb5873511e014207a475e2bb4e8b2e570d655b00ac19a9a0ca0a385ee/jsonschema-4.26.0-py3-none-any.whl", hash = "sha256:d489f15263b8d200f8387e64b4c3a75f06629559fb73deb8fdfb525f2dab50ce", size = 90630, upload-time = "2026-01-07T13:41:05.306Z" }, ] -[package.optional-dependencies] -format-nongpl = [ - { name = "fqdn" }, - { name = "idna" }, - { name = "isoduration" }, - { name = "jsonpointer" }, - { name = "rfc3339-validator" }, - { name = "rfc3986-validator" }, - { name = "rfc3987-syntax" }, - { name = "uri-template" }, - { name = "webcolors" }, -] - [[package]] name = "jsonschema-specifications" version = "2025.9.1" @@ -161,15 +100,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl", hash = "sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe", size = 18437, upload-time = "2025-09-08T01:34:57.871Z" }, ] -[[package]] -name = "lark" -version = "1.3.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/da/34/28fff3ab31ccff1fd4f6c7c7b0ceb2b6968d8ea4950663eadcb5720591a0/lark-1.3.1.tar.gz", hash = "sha256:b426a7a6d6d53189d318f2b6236ab5d6429eaf09259f1ca33eb716eed10d2905", size = 382732, upload-time = "2025-10-27T18:25:56.653Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/82/3d/14ce75ef66813643812f3093ab17e46d3a206942ce7376d31ec2d36229e7/lark-1.3.1-py3-none-any.whl", hash = "sha256:c629b661023a014c37da873b4ff58a817398d12635d3bbb2c5a03be7fe5d1e12", size = 113151, upload-time = "2025-10-27T18:25:54.882Z" }, -] - [[package]] name = "packaging" version = "26.3" @@ -296,18 +226,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, ] -[[package]] -name = "python-dateutil" -version = "2.9.0.post0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "six" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, -] - [[package]] name = "referencing" version = "0.37.0" @@ -343,18 +261,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9e/51/17023c0f8f1869d8806b979a2bffa3f861f26a3f1a66b094288323fba52f/rfc3986_validator-0.1.1-py2.py3-none-any.whl", hash = "sha256:2f235c432ef459970b4306369336b9d5dbdda31b510ca1e327636e01f528bfa9", size = 4242, upload-time = "2019-10-28T16:00:13.976Z" }, ] -[[package]] -name = "rfc3987-syntax" -version = "1.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "lark" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/2c/06/37c1a5557acf449e8e406a830a05bf885ac47d33270aec454ef78675008d/rfc3987_syntax-1.1.0.tar.gz", hash = "sha256:717a62cbf33cffdd16dfa3a497d81ce48a660ea691b1ddd7be710c22f00b4a0d", size = 14239, upload-time = "2025-07-18T01:05:05.015Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7e/71/44ce230e1b7fadd372515a97e32a83011f906ddded8d03e3c6aafbdedbb7/rfc3987_syntax-1.1.0-py3-none-any.whl", hash = "sha256:6c3d97604e4c5ce9f714898e05401a0445a641cfa276432b0a648c80856f6a3f", size = 8046, upload-time = "2025-07-18T01:05:03.843Z" }, -] - [[package]] name = "rfc8785" version = "0.1.4" @@ -504,30 +410,3 @@ sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3 wheels = [ { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, ] - -[[package]] -name = "tzdata" -version = "2026.5" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d9/68/f1b440335057bfce71b6e50a9d09445aa2ecbd08359a337976627b8409e7/tzdata-2026.5.tar.gz", hash = "sha256:8cc73c0a0bfca7dbfa59235d60b2eff82231dee33f53d206db1acd9173cfc0a7", size = 200404, upload-time = "2026-10-03T09:23:14.143Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/94/21/1e5995a1c920cce14e4bffae20c665ec10e7ed03ab25e006cd741092b718/tzdata-2026.5-py2.py3-none-any.whl", hash = "sha256:b683bd1b6659ddcd810ff02ad09ba821d4bf1065072805063eb35c49617905ac", size = 347996, upload-time = "2026-10-03T09:23:12.535Z" }, -] - -[[package]] -name = "uri-template" -version = "1.3.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/31/c7/0336f2bd0bcbada6ccef7aaa25e443c118a704f828a0620c6fa0207c1b64/uri-template-1.3.0.tar.gz", hash = "sha256:0e00f8eb65e18c7de20d595a14336e9f337ead580c70934141624b6d1ffdacc7", size = 21678, upload-time = "2023-06-21T01:49:05.374Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl", hash = "sha256:a44a133ea12d44a0c0f06d7d42a52d71282e77e2f937d8abd5655b8d56fc1363", size = 11140, upload-time = "2023-06-21T01:49:03.467Z" }, -] - -[[package]] -name = "webcolors" -version = "25.10.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/1d/7a/eb316761ec35664ea5174709a68bbd3389de60d4a1ebab8808bfc264ed67/webcolors-25.10.0.tar.gz", hash = "sha256:62abae86504f66d0f6364c2a8520de4a0c47b80c03fc3a5f1815fedbef7c19bf", size = 53491, upload-time = "2025-10-31T07:51:03.977Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl", hash = "sha256:032c727334856fc0b968f63daa252a1ac93d33db2f5267756623c210e57a4f1d", size = 14905, upload-time = "2025-10-31T07:51:01.778Z" }, -]