diff --git a/.github/workflows/copilot-setup-steps.yml b/.github/workflows/copilot-setup-steps.yml index 5bdb49e4..49f07221 100644 --- a/.github/workflows/copilot-setup-steps.yml +++ b/.github/workflows/copilot-setup-steps.yml @@ -25,14 +25,27 @@ jobs: # You can define any steps you want, and they will run before the agent starts. # If you do not check out your code, Copilot will do this for you. + env: + # Bump this string to force a toolchain cache bust (e.g. after a MoonBit release). + MOONBIT_CACHE_VERSION: "2026-06" + steps: - name: Checkout code uses: actions/checkout@v5 + - name: Cache MoonBit toolchain + id: toolchain-cache + uses: actions/cache@v5 + with: + path: ~/.moon + key: moonbit-toolchain-${{ runner.os }}-${{ env.MOONBIT_CACHE_VERSION }} + - name: Set up MoonBit - run: | - curl -fsSL https://cli.moonbitlang.com/install/unix.sh | bash - echo "$HOME/.moon/bin" >> "$GITHUB_PATH" + if: steps.toolchain-cache.outputs.cache-hit != 'true' + run: curl -fsSL https://cli.moonbitlang.com/install/unix.sh | bash + + - name: Add MoonBit to PATH + run: echo "$HOME/.moon/bin" >> "$GITHUB_PATH" - name: Update MoonBit dependencies run: | diff --git a/Makefile b/Makefile index 799569c3..50fd7290 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: build test bench-focus bench-focus-mbt subprocess-helpers-mbt-test architecture-audit architecture-boundary-audit architecture-boundary-audit-mbt architecture-boundary-audit-mbt-test architecture-state-audit architecture-state-audit-mbt architecture-state-audit-mbt-test test262 test262-metadata-test test262-metadata-mbt-test test262-metadata-tools-mbt-test test262-utils-test test262-utils-mbt-test test262-utils-corpus-mbt test262-runner-test test262-runner-mbt-test test262-runner-mbt test262-quick test262-filter test262-analyze test262-analyze-mbt test262-validate-skips test262-validate-skips-mbt test262-classify-by-edition-mbt classify-by-edition-mbt test262-download test262-report test262-report-test test262-report-mbt test262-skip-report unicode-tables unicode-tables-mbt clean +.PHONY: build test bench-focus bench-focus-mbt subprocess-helpers-mbt-test architecture-audit architecture-boundary-audit architecture-boundary-audit-mbt architecture-boundary-audit-mbt-test architecture-state-audit architecture-state-audit-mbt architecture-state-audit-mbt-test test262 test262-metadata-test test262-metadata-mbt-test test262-metadata-tools-mbt-test test262-utils-test test262-utils-mbt-test test262-utils-corpus-mbt test262-runner-test test262-runner-mbt-test test262-runner-mbt test262-quick test262-filter test262-analyze test262-analyze-mbt test262-validate-skips test262-validate-skips-mbt test262-classify-by-edition-mbt classify-by-edition-mbt test262-download test262-report test262-report-test test262-report-mbt test262-skip-report test262-feature-gap test262-feature-gap-test unicode-tables unicode-tables-mbt clean TEST262_COMMIT ?= main @@ -185,6 +185,18 @@ test262-skip-report: --metadata scripts/test262_skip_metadata.json \ --output docs/test262-skip-report.md +# Compare this repo's skip metadata against an external Test262 feature config. +# Pass EXT_CONFIG=/path/to/config.ini; output goes to docs/test262-feature-gap.md. +# This is a planning signal only — it does not run tests or report pass rates. +test262-feature-gap: + python3 scripts/test262_feature_gap.py \ + --ext-config $(EXT_CONFIG) \ + --output docs/test262-feature-gap.md + +# Run Python unit tests for the feature-gap comparison script. +test262-feature-gap-test: + python3 -m unittest scripts/test_test262_feature_gap.py -v + # Regenerate lexer/unicode_id.mbt from DerivedCoreProperties.txt. # Pass UNICODE_VERSION=X.Y.Z to target a specific Unicode release (default: 17.0.0). # MoonBit native is authoritative. diff --git a/docs/TEST262.md b/docs/TEST262.md index 9a122308..33788367 100644 --- a/docs/TEST262.md +++ b/docs/TEST262.md @@ -289,6 +289,45 @@ make test262-skip-report `make test262-skip-report` writes `docs/test262-skip-report.md`. +## Feature gap report + +To compare this repo's skip list against an external Test262 feature config (for +planning purposes only), use the feature-gap script: + +```bash +# Print to stdout +python3 scripts/test262_feature_gap.py --ext-config /path/to/external-config.ini + +# Write to docs/test262-feature-gap.md +make test262-feature-gap EXT_CONFIG=/path/to/external-config.ini +``` + +The script parses the `[features]` section of the external config. It handles +both bare feature tokens (external config runs those tests) and `feature=skip` +entries (external config explicitly skips those tests), then compares against +`scripts/test262_skip_metadata.json`. It produces a Markdown report with these +sections: + +- **Features we skip that the external config runs** — implementation gaps; + highest planning value. +- **Features we skip that the external config does not run** — split into two + sub-sections: features explicitly skipped by both configs (`feature=skip` + entries), and features we skip that aren't mentioned in the external config. +- **Features the external config runs that we also run** — no gap; included for + completeness. + +**This report is non-authoritative.** It compares metadata lists; it does not +execute tests, report pass rates, or claim conformance. Authoritative +conformance numbers come from CI Test262 artifacts (`make test262-report`). + +To run the Python unit tests for the script: + +```bash +make test262-feature-gap-test +# or directly: +python3 -m unittest scripts/test_test262_feature_gap.py -v +``` + ## CI Integration The GitHub Actions workflow (`.github/workflows/test262.yml`) runs the full suite automatically: diff --git a/scripts/test262_feature_gap.py b/scripts/test262_feature_gap.py new file mode 100644 index 00000000..721d5eaa --- /dev/null +++ b/scripts/test262_feature_gap.py @@ -0,0 +1,211 @@ +#!/usr/bin/env python3 +"""Compare this repo's Test262 skip metadata against an external feature config. + +This is a *planning signal* only. It does not execute tests, report pass rates, +or claim conformance. Authoritative conformance numbers come from CI Test262 +runs; see docs/TEST262.md and `make test262-report`. + +Usage: + python3 scripts/test262_feature_gap.py --ext-config /path/to/config.ini + python3 scripts/test262_feature_gap.py \ + --ext-config /path/to/config.ini \ + --metadata scripts/test262_skip_metadata.json \ + --output docs/test262-feature-gap.md +""" + +from __future__ import annotations + +import argparse +import json +import pathlib +import sys +from datetime import datetime, timezone + + +def parse_features_config(text: str) -> tuple[set[str], set[str]]: + """Parse a Test262 config's [features] section. + + Returns ``(runs, skips)`` where: + + - ``runs`` — features the external config runs (bare token, or ``=``). + - ``skips`` — features the external config explicitly marks as skipped + (``feature=skip``, case-insensitive). + + Inline comments (``#`` and ``//``) and blank lines are stripped. + Parsing stops at the next ``[…]`` section header. + """ + runs: set[str] = set() + skips: set[str] = set() + in_features = False + for raw_line in text.splitlines(): + line = raw_line.strip() + # Strip inline comments + for comment_char in ("#", "//"): + idx = line.find(comment_char) + if idx != -1: + line = line[:idx].strip() + if not line: + continue + if line.startswith("["): + in_features = line.lower() in ("[features]", "[ features ]") + continue + if not in_features: + continue + if "=" in line: + name, _, value = line.partition("=") + name = name.strip() + if not name: + continue + if value.strip().lower() == "skip": + skips.add(name) + else: + # Other values (e.g., =enabled, =on) — treat as runs. + runs.add(name) + else: + runs.add(line) + return runs, skips + + +def build_report( + our_skip: set[str], + ext_runs: set[str], + ext_skips: set[str], + ext_config_label: str, + now_iso: str | None = None, +) -> str: + """Return a Markdown feature-gap report.""" + now_iso = now_iso or datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + # Features the external config runs that we skip — highest planning value. + we_skip_they_run = sorted(our_skip & ext_runs) + # Features we skip that the external config explicitly marks as skipped. + both_explicitly_skip = sorted(our_skip & ext_skips) + # Features we skip that the external config doesn't mention at all. + we_skip_unmentioned = sorted(our_skip - ext_runs - ext_skips) + # Features the external config runs that aren't in our skip list. + ext_only = sorted(ext_runs - our_skip) + + lines: list[str] = [] + lines.append("# Test262 Feature Gap Report\n") + lines.append( + "> **This report is non-authoritative.** It compares skip metadata lists; " + "it does not execute tests or measure pass rates. " + "Authoritative conformance figures come from CI Test262 artifacts " + "(see `make test262-report` and `docs/TEST262.md`).\n" + ) + lines.append(f"Generated: {now_iso} ") + lines.append(f"External config: `{ext_config_label}` ") + lines.append(f"Our skip metadata: `scripts/test262_skip_metadata.json` \n") + + # Section 1 — planning priorities + lines.append( + f"## Features we skip that the external config runs ({len(we_skip_they_run)})\n" + ) + lines.append( + "These are implementation gaps relative to the external config. " + "Use as a planning signal only — the external config may have its own " + "limitations or different test coverage.\n" + ) + if we_skip_they_run: + for f in we_skip_they_run: + lines.append(f"- `{f}`") + else: + lines.append("*(none)*") + lines.append("") + + # Section 2 — features we skip that external also doesn't run + n_not_run = len(both_explicitly_skip) + len(we_skip_unmentioned) + lines.append(f"## Features we skip that the external config does not run ({n_not_run})\n") + + lines.append( + f"### Explicitly skipped by the external config ({len(both_explicitly_skip)})\n" + ) + if both_explicitly_skip: + for f in both_explicitly_skip: + lines.append(f"- `{f}`") + else: + lines.append("*(none)*") + lines.append("") + + lines.append( + f"### Not mentioned in the external config ({len(we_skip_unmentioned)})\n" + ) + lines.append( + "May be unimplemented, differently handled, or out of scope for the external config.\n" + ) + if we_skip_unmentioned: + for f in we_skip_unmentioned: + lines.append(f"- `{f}`") + else: + lines.append("*(none)*") + lines.append("") + + # Section 3 — no gap + lines.append( + f"## Features the external config runs that we also run ({len(ext_only)})\n" + ) + lines.append( + "These features are not in our skip list, so both configs run their tests.\n" + ) + if ext_only: + for f in ext_only: + lines.append(f"- `{f}`") + else: + lines.append("*(none)*") + lines.append("") + + return "\n".join(lines) + + +def main() -> int: + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument( + "--ext-config", + required=True, + metavar="FILE", + help="External Test262 config file with a [features] section", + ) + parser.add_argument( + "--metadata", + default=str(pathlib.Path(__file__).parent / "test262_skip_metadata.json"), + metavar="FILE", + help="Path to test262_skip_metadata.json (default: scripts/test262_skip_metadata.json)", + ) + parser.add_argument( + "--output", + metavar="FILE", + help="Write Markdown report to this file (default: stdout)", + ) + args = parser.parse_args() + + ext_path = pathlib.Path(args.ext_config) + if not ext_path.exists(): + print(f"error: external config not found: {ext_path}", file=sys.stderr) + return 1 + + meta_path = pathlib.Path(args.metadata) + if not meta_path.exists(): + print(f"error: skip metadata not found: {meta_path}", file=sys.stderr) + return 1 + + ext_runs, ext_skips = parse_features_config(ext_path.read_text(encoding="utf-8")) + data = json.loads(meta_path.read_text(encoding="utf-8")) + our_skip = set(data.get("skip_features", [])) + + report = build_report(our_skip, ext_runs, ext_skips, str(ext_path)) + + if args.output: + out_path = pathlib.Path(args.output) + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_text(report, encoding="utf-8") + print(f"Written to {out_path}") + else: + print(report, end="") + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/test_test262_feature_gap.py b/scripts/test_test262_feature_gap.py new file mode 100644 index 00000000..a785f9d0 --- /dev/null +++ b/scripts/test_test262_feature_gap.py @@ -0,0 +1,209 @@ +"""Tests for test262_feature_gap.py.""" + +from __future__ import annotations + +import pathlib +import re +import sys +import textwrap +import unittest + +# Ensure the scripts/ directory is on sys.path so this file can be run from +# either the repo root or the scripts/ directory directly. +sys.path.insert(0, str(pathlib.Path(__file__).parent)) +from test262_feature_gap import build_report, parse_features_config + +# Small fixture simulating a [features] section from an external Test262 config +# that uses both bare tokens (external runs) and token=skip (external skips). +FIXTURE_CONFIG = textwrap.dedent("""\ + # External Test262 config fixture + [features] + Array.fromAsync # external runs this + async-iteration=skip # external explicitly skips + class-fields-private=skip // external explicitly skips + BigInt=skip # external explicitly skips — was the P2 bug + Promise.allSettled # external runs this + regexp-lookbehind=skip # external explicitly skips + + [flags] + module +""") + +# A config with no [features] section. +CONFIG_NO_FEATURES = textwrap.dedent("""\ + [flags] + module + async +""") + +# A totally empty config. +CONFIG_EMPTY = "" + +OUR_SKIP = { + "async-iteration", + "BigInt", + "class-fields-private", + "regexp-lookbehind", + "Temporal", # only in our skip, not in external config + "WeakRef", # only in our skip, not in external config +} + + +class TestParseFeatures(unittest.TestCase): + def test_parses_bare_tokens_into_runs(self) -> None: + runs, _ = parse_features_config(FIXTURE_CONFIG) + self.assertIn("Array.fromAsync", runs) + self.assertIn("Promise.allSettled", runs) + + def test_skip_marker_excluded_from_runs(self) -> None: + # BigInt=skip must NOT appear in runs — this was the P2 bug. + runs, _ = parse_features_config(FIXTURE_CONFIG) + self.assertNotIn("BigInt", runs) + self.assertNotIn("BigInt=skip", runs) + + def test_skip_marker_included_in_skips(self) -> None: + # BigInt=skip must appear in skips with the bare feature name. + _, skips = parse_features_config(FIXTURE_CONFIG) + self.assertIn("BigInt", skips) + self.assertIn("async-iteration", skips) + self.assertIn("class-fields-private", skips) + self.assertIn("regexp-lookbehind", skips) + + def test_bare_tokens_not_in_skips(self) -> None: + _, skips = parse_features_config(FIXTURE_CONFIG) + self.assertNotIn("Array.fromAsync", skips) + self.assertNotIn("Promise.allSettled", skips) + + def test_non_skip_value_treated_as_run(self) -> None: + # A = other than "skip" should not be treated as a skip marker. + runs, skips = parse_features_config("[features]\nSomeFeature=enabled\n") + self.assertIn("SomeFeature", runs) + self.assertNotIn("SomeFeature", skips) + + def test_skip_case_insensitive(self) -> None: + # "=SKIP", "=Skip", "=skip" should all be treated as explicit skip. + _, skips = parse_features_config( + "[features]\nFoo=SKIP\nBar=Skip\nBaz=skip\n" + ) + self.assertIn("Foo", skips) + self.assertIn("Bar", skips) + self.assertIn("Baz", skips) + + def test_strips_inline_hash_comments(self) -> None: + runs, _ = parse_features_config(FIXTURE_CONFIG) + # "Array.fromAsync # external runs this" → "Array.fromAsync" + self.assertIn("Array.fromAsync", runs) + self.assertNotIn("Array.fromAsync # external runs this", runs) + + def test_strips_inline_slash_comments(self) -> None: + _, skips = parse_features_config(FIXTURE_CONFIG) + # "class-fields-private=skip // external explicitly skips" → skip + self.assertIn("class-fields-private", skips) + + def test_stops_at_next_section(self) -> None: + runs, skips = parse_features_config(FIXTURE_CONFIG) + # [flags] section items should not appear in either set. + self.assertNotIn("module", runs) + self.assertNotIn("module", skips) + + def test_empty_config_returns_empty_sets(self) -> None: + runs, skips = parse_features_config(CONFIG_EMPTY) + self.assertEqual(runs, set()) + self.assertEqual(skips, set()) + + def test_no_features_section_returns_empty_sets(self) -> None: + runs, skips = parse_features_config(CONFIG_NO_FEATURES) + self.assertEqual(runs, set()) + self.assertEqual(skips, set()) + + def test_blank_lines_ignored(self) -> None: + # Blank line between entries shouldn't produce empty-string feature. + runs, skips = parse_features_config(FIXTURE_CONFIG) + self.assertNotIn("", runs) + self.assertNotIn("", skips) + + def test_case_sensitive_feature_names(self) -> None: + # Feature names are case-sensitive in Test262 metadata. + runs, _ = parse_features_config("[features]\nBigInt\nbigint\n") + self.assertIn("BigInt", runs) + self.assertIn("bigint", runs) + + +class TestBuildReport(unittest.TestCase): + def _report(self, ext_runs: set[str], ext_skips: set[str] | None = None) -> str: + return build_report( + OUR_SKIP, + ext_runs, + ext_skips or set(), + "fixture.ini", + now_iso="2026-01-01T00:00:00Z", + ) + + def test_we_skip_they_run_section(self) -> None: + # Features in OUR_SKIP that are also in ext_runs belong in + # "Features we skip that the external config runs". + ext_runs = {"async-iteration", "class-fields-private", "BigInt"} + report = self._report(ext_runs) + self.assertIn("async-iteration", report) + self.assertIn("class-fields-private", report) + self.assertIn("BigInt", report) + + def test_skip_marker_appears_in_explicitly_skipped_subsection(self) -> None: + # BigInt from BigInt=skip should land in "Explicitly skipped by the + # external config", not in the "ext runs that we also run" section. + report = self._report(set(), {"BigInt", "async-iteration"}) + self.assertIn("Explicitly skipped by the external config", report) + self.assertIn("BigInt", report) + + def test_skip_marker_not_counted_in_ext_runs_section(self) -> None: + # =skip features must not appear under "the external config runs" — this was the bug. + report = self._report(set(), {"BigInt"}) + # The "Features we skip that the external config runs" section should show (0). + self.assertIn( + "Features we skip that the external config runs (0)", report + ) + + def test_unmentioned_features_go_to_not_mentioned_subsection(self) -> None: + # Temporal and WeakRef are in OUR_SKIP but absent from both ext_runs + # and ext_skips — they belong in "Not mentioned in the external config". + report = self._report(set(), set()) + self.assertIn("Not mentioned in the external config", report) + self.assertIn("Temporal", report) + self.assertIn("WeakRef", report) + + def test_non_authoritative_disclaimer_present(self) -> None: + report = self._report({"async-iteration"}) + self.assertIn("non-authoritative", report) + self.assertIn("does not execute tests", report) + + def test_ext_config_label_in_report(self) -> None: + report = self._report(set()) + self.assertIn("fixture.ini", report) + + def test_generated_timestamp_present(self) -> None: + report = self._report(set()) + self.assertIn("2026-01-01T00:00:00Z", report) + + def test_counts_in_section_headers(self) -> None: + # OUR_SKIP ∩ ext_runs = {async-iteration, BigInt} → (2) in the + # "we skip they run" header. + ext_runs = {"async-iteration", "BigInt"} + report = self._report(ext_runs) + self.assertIn("(2)", report) + + def test_empty_ext_config_shows_none_placeholder(self) -> None: + report = self._report(set()) + self.assertIn("*(none)*", report) + + def test_no_conformance_numbers_in_output(self) -> None: + # The report must not emit numeric conformance data. + report = self._report({"async-iteration"}) + for banned in ("P/E", "P/D", "Passed / Executed", "Passed / Discovered"): + self.assertNotIn(banned, report) + # No percentage figures in the data body (the disclaimer may mention + # "pass rates" as a concept but must not print numeric values). + self.assertIsNone(re.search(r"\d+\.\d+%", report)) + + +if __name__ == "__main__": + unittest.main()