From ddd09d53a109778291753eb5360683984c17faf4 Mon Sep 17 00:00:00 2001 From: Koji Ishimoto Date: Thu, 25 Jun 2026 23:28:37 +0900 Subject: [PATCH 1/3] ci: cache MoonBit toolchain in copilot-setup-steps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The toolchain cache pattern (cache restore → conditional install → unconditional PATH) was added to unit-test, test262, regression-check, bench, and startup-hyperfine, but copilot-setup-steps.yml still used the original unconditional curl | bash path. Apply the same three-step split here with a job-level env block carrying MOONBIT_CACHE_VERSION so the key can be busted on demand. Closes #292 Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/copilot-setup-steps.yml | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/.github/workflows/copilot-setup-steps.yml b/.github/workflows/copilot-setup-steps.yml index 5bdb49e4..49f07221 100644 --- a/.github/workflows/copilot-setup-steps.yml +++ b/.github/workflows/copilot-setup-steps.yml @@ -25,14 +25,27 @@ jobs: # You can define any steps you want, and they will run before the agent starts. # If you do not check out your code, Copilot will do this for you. + env: + # Bump this string to force a toolchain cache bust (e.g. after a MoonBit release). + MOONBIT_CACHE_VERSION: "2026-06" + steps: - name: Checkout code uses: actions/checkout@v5 + - name: Cache MoonBit toolchain + id: toolchain-cache + uses: actions/cache@v5 + with: + path: ~/.moon + key: moonbit-toolchain-${{ runner.os }}-${{ env.MOONBIT_CACHE_VERSION }} + - name: Set up MoonBit - run: | - curl -fsSL https://cli.moonbitlang.com/install/unix.sh | bash - echo "$HOME/.moon/bin" >> "$GITHUB_PATH" + if: steps.toolchain-cache.outputs.cache-hit != 'true' + run: curl -fsSL https://cli.moonbitlang.com/install/unix.sh | bash + + - name: Add MoonBit to PATH + run: echo "$HOME/.moon/bin" >> "$GITHUB_PATH" - name: Update MoonBit dependencies run: | From 7464065c9cc0f22270795854c98a97913597ba4f Mon Sep 17 00:00:00 2001 From: Koji Ishimoto Date: Thu, 25 Jun 2026 23:39:15 +0900 Subject: [PATCH 2/3] feat(scripts): add Test262 feature-gap comparison tool (#234) Add scripts/test262_feature_gap.py, a non-authoritative planning tool that compares this repo's skip metadata (scripts/test262_skip_metadata.json) against an external Test262 feature config's [features] section. The report has three sections: - Features we skip that the external config runs (planning priority signal) - Features both configs skip - Features the external config runs that we also run The script makes no conformance claims and emits no pass-rate numbers. A non-authoritative disclaimer appears at the top of every generated report. Also added: - scripts/test_test262_feature_gap.py: 15 Python unit tests covering the config parser and report builder (all passing) - Makefile targets: test262-feature-gap (requires EXT_CONFIG=) and test262-feature-gap-test (runs the Python unit tests) - docs/TEST262.md: "Feature gap report" section documenting usage Closes #234 Co-Authored-By: Claude Sonnet 4.6 --- Makefile | 14 ++- docs/TEST262.md | 37 ++++++ scripts/test262_feature_gap.py | 174 ++++++++++++++++++++++++++++ scripts/test_test262_feature_gap.py | 153 ++++++++++++++++++++++++ 4 files changed, 377 insertions(+), 1 deletion(-) create mode 100644 scripts/test262_feature_gap.py create mode 100644 scripts/test_test262_feature_gap.py diff --git a/Makefile b/Makefile index 799569c3..50fd7290 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: build test bench-focus bench-focus-mbt subprocess-helpers-mbt-test architecture-audit architecture-boundary-audit architecture-boundary-audit-mbt architecture-boundary-audit-mbt-test architecture-state-audit architecture-state-audit-mbt architecture-state-audit-mbt-test test262 test262-metadata-test test262-metadata-mbt-test test262-metadata-tools-mbt-test test262-utils-test test262-utils-mbt-test test262-utils-corpus-mbt test262-runner-test test262-runner-mbt-test test262-runner-mbt test262-quick test262-filter test262-analyze test262-analyze-mbt test262-validate-skips test262-validate-skips-mbt test262-classify-by-edition-mbt classify-by-edition-mbt test262-download test262-report test262-report-test test262-report-mbt test262-skip-report unicode-tables unicode-tables-mbt clean +.PHONY: build test bench-focus bench-focus-mbt subprocess-helpers-mbt-test architecture-audit architecture-boundary-audit architecture-boundary-audit-mbt architecture-boundary-audit-mbt-test architecture-state-audit architecture-state-audit-mbt architecture-state-audit-mbt-test test262 test262-metadata-test test262-metadata-mbt-test test262-metadata-tools-mbt-test test262-utils-test test262-utils-mbt-test test262-utils-corpus-mbt test262-runner-test test262-runner-mbt-test test262-runner-mbt test262-quick test262-filter test262-analyze test262-analyze-mbt test262-validate-skips test262-validate-skips-mbt test262-classify-by-edition-mbt classify-by-edition-mbt test262-download test262-report test262-report-test test262-report-mbt test262-skip-report test262-feature-gap test262-feature-gap-test unicode-tables unicode-tables-mbt clean TEST262_COMMIT ?= main @@ -185,6 +185,18 @@ test262-skip-report: --metadata scripts/test262_skip_metadata.json \ --output docs/test262-skip-report.md +# Compare this repo's skip metadata against an external Test262 feature config. +# Pass EXT_CONFIG=/path/to/config.ini; output goes to docs/test262-feature-gap.md. +# This is a planning signal only — it does not run tests or report pass rates. +test262-feature-gap: + python3 scripts/test262_feature_gap.py \ + --ext-config $(EXT_CONFIG) \ + --output docs/test262-feature-gap.md + +# Run Python unit tests for the feature-gap comparison script. +test262-feature-gap-test: + python3 -m unittest scripts/test_test262_feature_gap.py -v + # Regenerate lexer/unicode_id.mbt from DerivedCoreProperties.txt. # Pass UNICODE_VERSION=X.Y.Z to target a specific Unicode release (default: 17.0.0). # MoonBit native is authoritative. diff --git a/docs/TEST262.md b/docs/TEST262.md index 9a122308..4e5f8b51 100644 --- a/docs/TEST262.md +++ b/docs/TEST262.md @@ -289,6 +289,43 @@ make test262-skip-report `make test262-skip-report` writes `docs/test262-skip-report.md`. +## Feature gap report + +To compare this repo's skip list against an external Test262 feature config (for +planning purposes only), use the feature-gap script: + +```bash +# Print to stdout +python3 scripts/test262_feature_gap.py --ext-config /path/to/external-config.ini + +# Write to docs/test262-feature-gap.md +make test262-feature-gap EXT_CONFIG=/path/to/external-config.ini +``` + +The script parses the `[features]` section of the external config (an INI-style +file where each line inside `[features]` is a supported feature name) and +compares it against `scripts/test262_skip_metadata.json`. It produces a +Markdown report with three sections: + +- **Features we skip that the external config runs** — implementation gaps; + highest planning value. +- **Features both configs skip** — features not listed as supported by either + side. +- **Features the external config runs that we also run** — no gap; included for + completeness. + +**This report is non-authoritative.** It compares metadata lists; it does not +execute tests, report pass rates, or claim conformance. Authoritative +conformance numbers come from CI Test262 artifacts (`make test262-report`). + +To run the Python unit tests for the script: + +```bash +make test262-feature-gap-test +# or directly: +python3 -m unittest scripts/test_test262_feature_gap.py -v +``` + ## CI Integration The GitHub Actions workflow (`.github/workflows/test262.yml`) runs the full suite automatically: diff --git a/scripts/test262_feature_gap.py b/scripts/test262_feature_gap.py new file mode 100644 index 00000000..f220a6f1 --- /dev/null +++ b/scripts/test262_feature_gap.py @@ -0,0 +1,174 @@ +#!/usr/bin/env python3 +"""Compare this repo's Test262 skip metadata against an external feature config. + +This is a *planning signal* only. It does not execute tests, report pass rates, +or claim conformance. Authoritative conformance numbers come from CI Test262 +runs; see docs/TEST262.md and `make test262-report`. + +Usage: + python3 scripts/test262_feature_gap.py --ext-config /path/to/config.ini + python3 scripts/test262_feature_gap.py \ + --ext-config /path/to/config.ini \ + --metadata scripts/test262_skip_metadata.json \ + --output docs/test262-feature-gap.md +""" + +from __future__ import annotations + +import argparse +import json +import pathlib +import sys +from datetime import datetime, timezone + + +def parse_features_config(text: str) -> set[str]: + """Parse a Test262 config's [features] section into a set of feature names. + + Accepts INI-style files where feature names appear as bare tokens on their + own lines inside a ``[features]`` section. Inline comments (``#`` and + ``//``) and blank lines are stripped. Parsing stops at the next ``[…]`` + section header. + """ + features: set[str] = set() + in_features = False + for raw_line in text.splitlines(): + line = raw_line.strip() + # Strip inline comments + for comment_char in ("#", "//"): + idx = line.find(comment_char) + if idx != -1: + line = line[:idx].strip() + if not line: + continue + if line.startswith("["): + in_features = line.lower() in ("[features]", "[ features ]") + continue + if in_features: + features.add(line) + return features + + +def build_report( + our_skip: set[str], + ext_features: set[str], + ext_config_label: str, + now_iso: str | None = None, +) -> str: + """Return a Markdown feature-gap report.""" + now_iso = now_iso or datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + # Features the external config runs that we skip — highest planning value. + we_skip_they_run = sorted(our_skip & ext_features) + # Features both configs skip. + both_skip = sorted(our_skip - ext_features) + # Features in the external config that aren't in our skip list at all + # (i.e. features the external config runs and we also run — or we handle + # differently). Shown for completeness; low planning value. + ext_only = sorted(ext_features - our_skip) + + lines: list[str] = [] + lines.append("# Test262 Feature Gap Report\n") + lines.append( + "> **This report is non-authoritative.** It compares skip metadata lists; " + "it does not execute tests or measure pass rates. " + "Authoritative conformance figures come from CI Test262 artifacts " + "(see `make test262-report` and `docs/TEST262.md`).\n" + ) + lines.append(f"Generated: {now_iso} ") + lines.append(f"External config: `{ext_config_label}` ") + lines.append( + f"Our skip metadata: `scripts/test262_skip_metadata.json` \n" + ) + + lines.append( + f"## Features we skip that the external config runs ({len(we_skip_they_run)})\n" + ) + lines.append( + "These are implementation gaps relative to the external config. " + "Use as a planning signal only — the external config may have its own " + "limitations or different test coverage.\n" + ) + if we_skip_they_run: + for f in we_skip_they_run: + lines.append(f"- `{f}`") + else: + lines.append("*(none)*") + lines.append("") + + lines.append(f"## Features both configs skip ({len(both_skip)})\n") + if both_skip: + for f in both_skip: + lines.append(f"- `{f}`") + else: + lines.append("*(none)*") + lines.append("") + + lines.append( + f"## Features the external config runs that we also run ({len(ext_only)})\n" + ) + lines.append( + "These features are not in our skip list, so both configs run their tests.\n" + ) + if ext_only: + for f in ext_only: + lines.append(f"- `{f}`") + else: + lines.append("*(none)*") + lines.append("") + + return "\n".join(lines) + + +def main() -> int: + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument( + "--ext-config", + required=True, + metavar="FILE", + help="External Test262 config file with a [features] section", + ) + parser.add_argument( + "--metadata", + default=str(pathlib.Path(__file__).parent / "test262_skip_metadata.json"), + metavar="FILE", + help="Path to test262_skip_metadata.json (default: scripts/test262_skip_metadata.json)", + ) + parser.add_argument( + "--output", + metavar="FILE", + help="Write Markdown report to this file (default: stdout)", + ) + args = parser.parse_args() + + ext_path = pathlib.Path(args.ext_config) + if not ext_path.exists(): + print(f"error: external config not found: {ext_path}", file=sys.stderr) + return 1 + + meta_path = pathlib.Path(args.metadata) + if not meta_path.exists(): + print(f"error: skip metadata not found: {meta_path}", file=sys.stderr) + return 1 + + ext_features = parse_features_config(ext_path.read_text(encoding="utf-8")) + data = json.loads(meta_path.read_text(encoding="utf-8")) + our_skip = set(data.get("skip_features", [])) + + report = build_report(our_skip, ext_features, str(ext_path)) + + if args.output: + out_path = pathlib.Path(args.output) + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_text(report, encoding="utf-8") + print(f"Written to {out_path}") + else: + print(report, end="") + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/test_test262_feature_gap.py b/scripts/test_test262_feature_gap.py new file mode 100644 index 00000000..63b65436 --- /dev/null +++ b/scripts/test_test262_feature_gap.py @@ -0,0 +1,153 @@ +"""Tests for test262_feature_gap.py.""" + +from __future__ import annotations + +import pathlib +import sys +import textwrap +import unittest + +# Ensure the scripts/ directory is on sys.path so this file can be run from +# either the repo root or the scripts/ directory directly. +sys.path.insert(0, str(pathlib.Path(__file__).parent)) +from test262_feature_gap import build_report, parse_features_config + +# Small fixture simulating a [features] section from an external Test262 config. +FIXTURE_CONFIG = textwrap.dedent("""\ + # External Test262 config fixture + [features] + Array.fromAsync # ES2024 + async-iteration # skip in both + class-fields-private // both skip + BigInt # both skip + Promise.allSettled + regexp-lookbehind # both skip + + [flags] + module +""") + +# A config with no [features] section. +CONFIG_NO_FEATURES = textwrap.dedent("""\ + [flags] + module + async +""") + +# A totally empty config. +CONFIG_EMPTY = "" + +OUR_SKIP = { + "async-iteration", + "BigInt", + "class-fields-private", + "regexp-lookbehind", + "Temporal", # only in our skip, not in external config + "WeakRef", # only in our skip, not in external config +} + + +class TestParseFeatures(unittest.TestCase): + def test_parses_features_section(self) -> None: + result = parse_features_config(FIXTURE_CONFIG) + self.assertIn("async-iteration", result) + self.assertIn("class-fields-private", result) + self.assertIn("BigInt", result) + self.assertIn("Promise.allSettled", result) + self.assertIn("Array.fromAsync", result) + self.assertIn("regexp-lookbehind", result) + + def test_strips_inline_hash_comments(self) -> None: + result = parse_features_config(FIXTURE_CONFIG) + # "Array.fromAsync # ES2024" → "Array.fromAsync" + self.assertIn("Array.fromAsync", result) + self.assertNotIn("Array.fromAsync # ES2024", result) + + def test_strips_inline_slash_comments(self) -> None: + result = parse_features_config(FIXTURE_CONFIG) + # "class-fields-private // both skip" → "class-fields-private" + self.assertIn("class-fields-private", result) + + def test_stops_at_next_section(self) -> None: + result = parse_features_config(FIXTURE_CONFIG) + # [flags] section items should not appear + self.assertNotIn("module", result) + + def test_empty_config_returns_empty_set(self) -> None: + self.assertEqual(parse_features_config(CONFIG_EMPTY), set()) + + def test_no_features_section_returns_empty_set(self) -> None: + self.assertEqual(parse_features_config(CONFIG_NO_FEATURES), set()) + + def test_blank_lines_ignored(self) -> None: + # Blank line between entries shouldn't produce empty-string feature. + result = parse_features_config(FIXTURE_CONFIG) + self.assertNotIn("", result) + + def test_case_sensitive(self) -> None: + # Feature names are case-sensitive in Test262 metadata. + result = parse_features_config("[features]\nBigInt\nbigint\n") + self.assertIn("BigInt", result) + self.assertIn("bigint", result) + + +class TestBuildReport(unittest.TestCase): + def _report(self, ext_features: set[str]) -> str: + return build_report( + OUR_SKIP, ext_features, "fixture.ini", now_iso="2026-01-01T00:00:00Z" + ) + + def test_we_skip_they_run_section(self) -> None: + # async-iteration, class-fields-private, BigInt, regexp-lookbehind all + # appear in both OUR_SKIP and the fixture ext_features, so they belong + # in "both skip", not "we skip they run". Only Temporal/WeakRef are + # purely ours. + ext = {"async-iteration", "class-fields-private", "BigInt"} + # Temporal and WeakRef in OUR_SKIP but NOT in ext → "we skip they run" + # would be empty because we're comparing OUR_SKIP ∩ ext (they run what we skip) + # Actually: "we skip they run" = OUR_SKIP ∩ ext_features + # So ext={"async-iteration"...} means those are "we skip AND they run" + report = build_report( + OUR_SKIP, ext, "fixture.ini", now_iso="2026-01-01T00:00:00Z" + ) + self.assertIn("async-iteration", report) + self.assertIn("class-fields-private", report) + + def test_non_authoritative_disclaimer_present(self) -> None: + report = self._report({"async-iteration"}) + self.assertIn("non-authoritative", report) + self.assertIn("does not execute tests", report) + + def test_ext_config_label_in_report(self) -> None: + report = self._report(set()) + self.assertIn("fixture.ini", report) + + def test_generated_timestamp_present(self) -> None: + report = self._report(set()) + self.assertIn("2026-01-01T00:00:00Z", report) + + def test_counts_in_section_headers(self) -> None: + ext = {"async-iteration", "BigInt"} # both in OUR_SKIP → "we skip they run" + report = build_report( + OUR_SKIP, ext, "fixture.ini", now_iso="2026-01-01T00:00:00Z" + ) + # "we skip they run" = OUR_SKIP ∩ ext = {async-iteration, BigInt} + self.assertIn("(2)", report) + + def test_empty_ext_config(self) -> None: + report = self._report(set()) + self.assertIn("*(none)*", report) + + def test_no_conformance_numbers_in_output(self) -> None: + # The report must not emit numeric conformance data. + report = self._report({"async-iteration"}) + for banned in ("P/E", "P/D", "Passed / Executed", "Passed / Discovered"): + self.assertNotIn(banned, report) + # No percentage figures in the data body (the disclaimer may mention + # "pass rates" as a concept but must not print numeric values). + import re + self.assertIsNone(re.search(r"\d+\.\d+%", report)) + + +if __name__ == "__main__": + unittest.main() From 50f81f6bb16a2ff0f7ae45e66f981c118733647d Mon Sep 17 00:00:00 2001 From: Koji Ishimoto Date: Thu, 25 Jun 2026 23:49:41 +0900 Subject: [PATCH 3/3] fix(scripts): parse feature=skip entries in external Test262 configs (#234) External configs like QuickJS's test262.conf use `feature=skip` inside [features] to mark unsupported features. The old parser treated the whole token as a feature name, placing `BigInt=skip` in the "external config runs" set and causing materially misleading planning reports. parse_features_config now returns (runs, skips): bare tokens go to runs, `feature=skip` entries go to skips. build_report splits the "both skip" section into two sub-sections: explicitly skipped by external vs not mentioned. 23 tests (up from 15), including a regression test for the =skip misclassification. Co-Authored-By: Claude Sonnet 4.6 --- docs/TEST262.md | 14 ++- scripts/test262_feature_gap.py | 89 ++++++++++----- scripts/test_test262_feature_gap.py | 168 ++++++++++++++++++---------- 3 files changed, 183 insertions(+), 88 deletions(-) diff --git a/docs/TEST262.md b/docs/TEST262.md index 4e5f8b51..33788367 100644 --- a/docs/TEST262.md +++ b/docs/TEST262.md @@ -302,15 +302,17 @@ python3 scripts/test262_feature_gap.py --ext-config /path/to/external-config.ini make test262-feature-gap EXT_CONFIG=/path/to/external-config.ini ``` -The script parses the `[features]` section of the external config (an INI-style -file where each line inside `[features]` is a supported feature name) and -compares it against `scripts/test262_skip_metadata.json`. It produces a -Markdown report with three sections: +The script parses the `[features]` section of the external config. It handles +both bare feature tokens (external config runs those tests) and `feature=skip` +entries (external config explicitly skips those tests), then compares against +`scripts/test262_skip_metadata.json`. It produces a Markdown report with these +sections: - **Features we skip that the external config runs** — implementation gaps; highest planning value. -- **Features both configs skip** — features not listed as supported by either - side. +- **Features we skip that the external config does not run** — split into two + sub-sections: features explicitly skipped by both configs (`feature=skip` + entries), and features we skip that aren't mentioned in the external config. - **Features the external config runs that we also run** — no gap; included for completeness. diff --git a/scripts/test262_feature_gap.py b/scripts/test262_feature_gap.py index f220a6f1..721d5eaa 100644 --- a/scripts/test262_feature_gap.py +++ b/scripts/test262_feature_gap.py @@ -22,15 +22,20 @@ from datetime import datetime, timezone -def parse_features_config(text: str) -> set[str]: - """Parse a Test262 config's [features] section into a set of feature names. +def parse_features_config(text: str) -> tuple[set[str], set[str]]: + """Parse a Test262 config's [features] section. - Accepts INI-style files where feature names appear as bare tokens on their - own lines inside a ``[features]`` section. Inline comments (``#`` and - ``//``) and blank lines are stripped. Parsing stops at the next ``[…]`` - section header. + Returns ``(runs, skips)`` where: + + - ``runs`` — features the external config runs (bare token, or ``=``). + - ``skips`` — features the external config explicitly marks as skipped + (``feature=skip``, case-insensitive). + + Inline comments (``#`` and ``//``) and blank lines are stripped. + Parsing stops at the next ``[…]`` section header. """ - features: set[str] = set() + runs: set[str] = set() + skips: set[str] = set() in_features = False for raw_line in text.splitlines(): line = raw_line.strip() @@ -44,14 +49,27 @@ def parse_features_config(text: str) -> set[str]: if line.startswith("["): in_features = line.lower() in ("[features]", "[ features ]") continue - if in_features: - features.add(line) - return features + if not in_features: + continue + if "=" in line: + name, _, value = line.partition("=") + name = name.strip() + if not name: + continue + if value.strip().lower() == "skip": + skips.add(name) + else: + # Other values (e.g., =enabled, =on) — treat as runs. + runs.add(name) + else: + runs.add(line) + return runs, skips def build_report( our_skip: set[str], - ext_features: set[str], + ext_runs: set[str], + ext_skips: set[str], ext_config_label: str, now_iso: str | None = None, ) -> str: @@ -59,13 +77,13 @@ def build_report( now_iso = now_iso or datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") # Features the external config runs that we skip — highest planning value. - we_skip_they_run = sorted(our_skip & ext_features) - # Features both configs skip. - both_skip = sorted(our_skip - ext_features) - # Features in the external config that aren't in our skip list at all - # (i.e. features the external config runs and we also run — or we handle - # differently). Shown for completeness; low planning value. - ext_only = sorted(ext_features - our_skip) + we_skip_they_run = sorted(our_skip & ext_runs) + # Features we skip that the external config explicitly marks as skipped. + both_explicitly_skip = sorted(our_skip & ext_skips) + # Features we skip that the external config doesn't mention at all. + we_skip_unmentioned = sorted(our_skip - ext_runs - ext_skips) + # Features the external config runs that aren't in our skip list. + ext_only = sorted(ext_runs - our_skip) lines: list[str] = [] lines.append("# Test262 Feature Gap Report\n") @@ -77,10 +95,9 @@ def build_report( ) lines.append(f"Generated: {now_iso} ") lines.append(f"External config: `{ext_config_label}` ") - lines.append( - f"Our skip metadata: `scripts/test262_skip_metadata.json` \n" - ) + lines.append(f"Our skip metadata: `scripts/test262_skip_metadata.json` \n") + # Section 1 — planning priorities lines.append( f"## Features we skip that the external config runs ({len(we_skip_they_run)})\n" ) @@ -96,14 +113,34 @@ def build_report( lines.append("*(none)*") lines.append("") - lines.append(f"## Features both configs skip ({len(both_skip)})\n") - if both_skip: - for f in both_skip: + # Section 2 — features we skip that external also doesn't run + n_not_run = len(both_explicitly_skip) + len(we_skip_unmentioned) + lines.append(f"## Features we skip that the external config does not run ({n_not_run})\n") + + lines.append( + f"### Explicitly skipped by the external config ({len(both_explicitly_skip)})\n" + ) + if both_explicitly_skip: + for f in both_explicitly_skip: + lines.append(f"- `{f}`") + else: + lines.append("*(none)*") + lines.append("") + + lines.append( + f"### Not mentioned in the external config ({len(we_skip_unmentioned)})\n" + ) + lines.append( + "May be unimplemented, differently handled, or out of scope for the external config.\n" + ) + if we_skip_unmentioned: + for f in we_skip_unmentioned: lines.append(f"- `{f}`") else: lines.append("*(none)*") lines.append("") + # Section 3 — no gap lines.append( f"## Features the external config runs that we also run ({len(ext_only)})\n" ) @@ -153,11 +190,11 @@ def main() -> int: print(f"error: skip metadata not found: {meta_path}", file=sys.stderr) return 1 - ext_features = parse_features_config(ext_path.read_text(encoding="utf-8")) + ext_runs, ext_skips = parse_features_config(ext_path.read_text(encoding="utf-8")) data = json.loads(meta_path.read_text(encoding="utf-8")) our_skip = set(data.get("skip_features", [])) - report = build_report(our_skip, ext_features, str(ext_path)) + report = build_report(our_skip, ext_runs, ext_skips, str(ext_path)) if args.output: out_path = pathlib.Path(args.output) diff --git a/scripts/test_test262_feature_gap.py b/scripts/test_test262_feature_gap.py index 63b65436..a785f9d0 100644 --- a/scripts/test_test262_feature_gap.py +++ b/scripts/test_test262_feature_gap.py @@ -3,6 +3,7 @@ from __future__ import annotations import pathlib +import re import sys import textwrap import unittest @@ -12,16 +13,17 @@ sys.path.insert(0, str(pathlib.Path(__file__).parent)) from test262_feature_gap import build_report, parse_features_config -# Small fixture simulating a [features] section from an external Test262 config. +# Small fixture simulating a [features] section from an external Test262 config +# that uses both bare tokens (external runs) and token=skip (external skips). FIXTURE_CONFIG = textwrap.dedent("""\ # External Test262 config fixture [features] - Array.fromAsync # ES2024 - async-iteration # skip in both - class-fields-private // both skip - BigInt # both skip - Promise.allSettled - regexp-lookbehind # both skip + Array.fromAsync # external runs this + async-iteration=skip # external explicitly skips + class-fields-private=skip // external explicitly skips + BigInt=skip # external explicitly skips — was the P2 bug + Promise.allSettled # external runs this + regexp-lookbehind=skip # external explicitly skips [flags] module @@ -48,70 +50,126 @@ class TestParseFeatures(unittest.TestCase): - def test_parses_features_section(self) -> None: - result = parse_features_config(FIXTURE_CONFIG) - self.assertIn("async-iteration", result) - self.assertIn("class-fields-private", result) - self.assertIn("BigInt", result) - self.assertIn("Promise.allSettled", result) - self.assertIn("Array.fromAsync", result) - self.assertIn("regexp-lookbehind", result) + def test_parses_bare_tokens_into_runs(self) -> None: + runs, _ = parse_features_config(FIXTURE_CONFIG) + self.assertIn("Array.fromAsync", runs) + self.assertIn("Promise.allSettled", runs) + + def test_skip_marker_excluded_from_runs(self) -> None: + # BigInt=skip must NOT appear in runs — this was the P2 bug. + runs, _ = parse_features_config(FIXTURE_CONFIG) + self.assertNotIn("BigInt", runs) + self.assertNotIn("BigInt=skip", runs) + + def test_skip_marker_included_in_skips(self) -> None: + # BigInt=skip must appear in skips with the bare feature name. + _, skips = parse_features_config(FIXTURE_CONFIG) + self.assertIn("BigInt", skips) + self.assertIn("async-iteration", skips) + self.assertIn("class-fields-private", skips) + self.assertIn("regexp-lookbehind", skips) + + def test_bare_tokens_not_in_skips(self) -> None: + _, skips = parse_features_config(FIXTURE_CONFIG) + self.assertNotIn("Array.fromAsync", skips) + self.assertNotIn("Promise.allSettled", skips) + + def test_non_skip_value_treated_as_run(self) -> None: + # A = other than "skip" should not be treated as a skip marker. + runs, skips = parse_features_config("[features]\nSomeFeature=enabled\n") + self.assertIn("SomeFeature", runs) + self.assertNotIn("SomeFeature", skips) + + def test_skip_case_insensitive(self) -> None: + # "=SKIP", "=Skip", "=skip" should all be treated as explicit skip. + _, skips = parse_features_config( + "[features]\nFoo=SKIP\nBar=Skip\nBaz=skip\n" + ) + self.assertIn("Foo", skips) + self.assertIn("Bar", skips) + self.assertIn("Baz", skips) def test_strips_inline_hash_comments(self) -> None: - result = parse_features_config(FIXTURE_CONFIG) - # "Array.fromAsync # ES2024" → "Array.fromAsync" - self.assertIn("Array.fromAsync", result) - self.assertNotIn("Array.fromAsync # ES2024", result) + runs, _ = parse_features_config(FIXTURE_CONFIG) + # "Array.fromAsync # external runs this" → "Array.fromAsync" + self.assertIn("Array.fromAsync", runs) + self.assertNotIn("Array.fromAsync # external runs this", runs) def test_strips_inline_slash_comments(self) -> None: - result = parse_features_config(FIXTURE_CONFIG) - # "class-fields-private // both skip" → "class-fields-private" - self.assertIn("class-fields-private", result) + _, skips = parse_features_config(FIXTURE_CONFIG) + # "class-fields-private=skip // external explicitly skips" → skip + self.assertIn("class-fields-private", skips) def test_stops_at_next_section(self) -> None: - result = parse_features_config(FIXTURE_CONFIG) - # [flags] section items should not appear - self.assertNotIn("module", result) + runs, skips = parse_features_config(FIXTURE_CONFIG) + # [flags] section items should not appear in either set. + self.assertNotIn("module", runs) + self.assertNotIn("module", skips) - def test_empty_config_returns_empty_set(self) -> None: - self.assertEqual(parse_features_config(CONFIG_EMPTY), set()) + def test_empty_config_returns_empty_sets(self) -> None: + runs, skips = parse_features_config(CONFIG_EMPTY) + self.assertEqual(runs, set()) + self.assertEqual(skips, set()) - def test_no_features_section_returns_empty_set(self) -> None: - self.assertEqual(parse_features_config(CONFIG_NO_FEATURES), set()) + def test_no_features_section_returns_empty_sets(self) -> None: + runs, skips = parse_features_config(CONFIG_NO_FEATURES) + self.assertEqual(runs, set()) + self.assertEqual(skips, set()) def test_blank_lines_ignored(self) -> None: # Blank line between entries shouldn't produce empty-string feature. - result = parse_features_config(FIXTURE_CONFIG) - self.assertNotIn("", result) + runs, skips = parse_features_config(FIXTURE_CONFIG) + self.assertNotIn("", runs) + self.assertNotIn("", skips) - def test_case_sensitive(self) -> None: + def test_case_sensitive_feature_names(self) -> None: # Feature names are case-sensitive in Test262 metadata. - result = parse_features_config("[features]\nBigInt\nbigint\n") - self.assertIn("BigInt", result) - self.assertIn("bigint", result) + runs, _ = parse_features_config("[features]\nBigInt\nbigint\n") + self.assertIn("BigInt", runs) + self.assertIn("bigint", runs) class TestBuildReport(unittest.TestCase): - def _report(self, ext_features: set[str]) -> str: + def _report(self, ext_runs: set[str], ext_skips: set[str] | None = None) -> str: return build_report( - OUR_SKIP, ext_features, "fixture.ini", now_iso="2026-01-01T00:00:00Z" + OUR_SKIP, + ext_runs, + ext_skips or set(), + "fixture.ini", + now_iso="2026-01-01T00:00:00Z", ) def test_we_skip_they_run_section(self) -> None: - # async-iteration, class-fields-private, BigInt, regexp-lookbehind all - # appear in both OUR_SKIP and the fixture ext_features, so they belong - # in "both skip", not "we skip they run". Only Temporal/WeakRef are - # purely ours. - ext = {"async-iteration", "class-fields-private", "BigInt"} - # Temporal and WeakRef in OUR_SKIP but NOT in ext → "we skip they run" - # would be empty because we're comparing OUR_SKIP ∩ ext (they run what we skip) - # Actually: "we skip they run" = OUR_SKIP ∩ ext_features - # So ext={"async-iteration"...} means those are "we skip AND they run" - report = build_report( - OUR_SKIP, ext, "fixture.ini", now_iso="2026-01-01T00:00:00Z" - ) + # Features in OUR_SKIP that are also in ext_runs belong in + # "Features we skip that the external config runs". + ext_runs = {"async-iteration", "class-fields-private", "BigInt"} + report = self._report(ext_runs) self.assertIn("async-iteration", report) self.assertIn("class-fields-private", report) + self.assertIn("BigInt", report) + + def test_skip_marker_appears_in_explicitly_skipped_subsection(self) -> None: + # BigInt from BigInt=skip should land in "Explicitly skipped by the + # external config", not in the "ext runs that we also run" section. + report = self._report(set(), {"BigInt", "async-iteration"}) + self.assertIn("Explicitly skipped by the external config", report) + self.assertIn("BigInt", report) + + def test_skip_marker_not_counted_in_ext_runs_section(self) -> None: + # =skip features must not appear under "the external config runs" — this was the bug. + report = self._report(set(), {"BigInt"}) + # The "Features we skip that the external config runs" section should show (0). + self.assertIn( + "Features we skip that the external config runs (0)", report + ) + + def test_unmentioned_features_go_to_not_mentioned_subsection(self) -> None: + # Temporal and WeakRef are in OUR_SKIP but absent from both ext_runs + # and ext_skips — they belong in "Not mentioned in the external config". + report = self._report(set(), set()) + self.assertIn("Not mentioned in the external config", report) + self.assertIn("Temporal", report) + self.assertIn("WeakRef", report) def test_non_authoritative_disclaimer_present(self) -> None: report = self._report({"async-iteration"}) @@ -127,14 +185,13 @@ def test_generated_timestamp_present(self) -> None: self.assertIn("2026-01-01T00:00:00Z", report) def test_counts_in_section_headers(self) -> None: - ext = {"async-iteration", "BigInt"} # both in OUR_SKIP → "we skip they run" - report = build_report( - OUR_SKIP, ext, "fixture.ini", now_iso="2026-01-01T00:00:00Z" - ) - # "we skip they run" = OUR_SKIP ∩ ext = {async-iteration, BigInt} + # OUR_SKIP ∩ ext_runs = {async-iteration, BigInt} → (2) in the + # "we skip they run" header. + ext_runs = {"async-iteration", "BigInt"} + report = self._report(ext_runs) self.assertIn("(2)", report) - def test_empty_ext_config(self) -> None: + def test_empty_ext_config_shows_none_placeholder(self) -> None: report = self._report(set()) self.assertIn("*(none)*", report) @@ -145,7 +202,6 @@ def test_no_conformance_numbers_in_output(self) -> None: self.assertNotIn(banned, report) # No percentage figures in the data body (the disclaimer may mention # "pass rates" as a concept but must not print numeric values). - import re self.assertIsNone(re.search(r"\d+\.\d+%", report))