diff --git a/docs/results.md b/docs/results.md index 77a4886..f399348 100644 --- a/docs/results.md +++ b/docs/results.md @@ -4,6 +4,24 @@ date = 2024-12-10T07:07:07+01:00 description = 'Fetch results from the KernelCI ecosystem.' +++ +## Regression comparison and CI gates + +```shell +kci-dev results compare --giturl URL --branch BRANCH --format json BASE HEAD +kci-dev results gate --giturl URL --branch BRANCH --base BASE --head HEAD \ + --fail-on regression --format json +kci-dev results compare --giturl URL --branch BRANCH --include-issues BASE HEAD +``` + +Reports classify executions as `regression`, `fixed`, `unstable`, +`persistent_fail`, `new`, or `missing`, while preserving duplicates. Identity +includes origin, platform, architecture, compiler, configuration, and path. +Exit status is 0 without a policy violation, 1 for a policy violation, and 2 +when API or infrastructure failures make the result incomplete. + +Known-issue lookup is opt-in with `--include-issues`, since it makes one +additional Dashboard request for each regression and persistent failure. + `kci-dev` pulls from our Dashboard API. As of now, it is an EXPERIMENTAL tooling under development with close collaboration from Linux kernel maintainers. > KNOWN ISSUE: The Dashboard endpoint we are using returns a file of a few megabytes in size, so download may take diff --git a/kcidev/api.py b/kcidev/api.py index 0067b23..d7d6192 100644 --- a/kcidev/api.py +++ b/kcidev/api.py @@ -55,6 +55,7 @@ send_jobretry, send_patchset, ) +from kcidev.libs.regression import RegressionReport from kcidev.main import get_cli @@ -419,22 +420,22 @@ def get_hardware_tests(self, name, origin): True, ) - def get_build_issues(self, build_id): + def get_build_issues(self, build_id, error_verbose=True): return self._dashboard_request( "Dashboard build issues request failed", dashboard_fetch_build_issues, build_id, True, - True, + error_verbose, ) - def get_boot_issues(self, test_id): + def get_boot_issues(self, test_id, error_verbose=True): return self._dashboard_request( "Dashboard boot issues request failed", dashboard_fetch_boot_issues, test_id, True, - True, + error_verbose, ) def get_issue_list(self, origin=None, days=7): @@ -484,8 +485,8 @@ def get_tree_report( git_url, test_path=None, history_size=10, - max_age_in_hours=None, - min_age_in_hours=None, + max_age_in_hours=24, + min_age_in_hours=0, ): return self._dashboard_request( "Dashboard tree report request failed", @@ -500,6 +501,68 @@ def get_tree_report( min_age_in_hours, ) + def compare_results( + self, base, head, giturl, branch, origin="maestro", include_issues=False + ): + """Compare two dashboard checkouts and return a CI-grade report dict. + + Issue lookup is disabled by default because it requires one additional + Dashboard request for every regression and persistent failure. + """ + + def checkout(commit): + return { + "builds": self.get_builds(origin, giturl, branch, commit).get( + "builds", [] + ), + "boots": self.get_boots(origin, giturl, branch, commit).get( + "boots", [] + ), + "tests": self.get_tests(origin, giturl, branch, commit).get( + "tests", [] + ), + } + + base_results, head_results = checkout(base), checkout(head) + # tree-report supplies history-aware unstable/regression decisions. If + # HEAD is not the newest checkout the raw transition remains useful. + history_incomplete = False + try: + history = self.get_tree_report(origin, branch, giturl) + except KciDevError: + # History improves classification, but is not required to compare + # the two explicitly requested checkouts. + history = None + history_incomplete = True + report = RegressionReport.compare( + base, head, base_results, head_results, history + ) + report.incomplete = history_incomplete + if include_issues: + for item in report.items: + if item["category"] not in ("regression", "persistent_fail"): + continue + result_id = item["head_id"] + if not result_id: + continue + try: + issues = ( + self.get_build_issues(result_id, error_verbose=False) + if item["identity"]["kind"] == "build" + else self.get_boot_issues(result_id, error_verbose=False) + ) + except KciDevError as exc: + if "no issues" in str(exc).lower(): + issues = [] + else: + report.incomplete = True + continue + item["known_issues"] = [ + issue.get("id", issue) if isinstance(issue, dict) else issue + for issue in issues + ] + return report.to_dict() + def _instance_setting(self, key, *, human_readable_key=None): value = ((self.cfg or {}).get(self.instance) or {}).get(key) if not value: diff --git a/kcidev/libs/regression.py b/kcidev/libs/regression.py new file mode 100644 index 0000000..2461a0d --- /dev/null +++ b/kcidev/libs/regression.py @@ -0,0 +1,169 @@ +"""Pure regression classification and reporting. + +This module deliberately has no Click or HTTP dependencies. It can therefore +be used by the command line, the public Python client, and the MCP server. +""" + +from collections import Counter, defaultdict +from dataclasses import dataclass, field + +CATEGORIES = ("regression", "fixed", "unstable", "persistent_fail", "new", "missing") +FAIL_STATUSES = {"FAIL", "ERROR"} + + +def _value(item, *names, default="unknown"): + for name in names: + value = item.get(name) + if value not in (None, ""): + return value + return default + + +def result_identity(item, kind="test"): + """Return the complete, stable identity of a result (never its result id).""" + environment = item.get("environment_misc") or {} + return { + "kind": kind, + "origin": _value(item, "origin"), + "platform": _value( + item, "platform", "hardware", default=environment.get("platform", "unknown") + ), + "architecture": _value(item, "architecture", "arch"), + "compiler": _value(item, "compiler"), + "config": _value(item, "config", "config_name"), + "path": _value( + item, "path", "test_path", default="build" if kind == "build" else "unknown" + ), + } + + +def _key(item, kind): + identity = result_identity(item, kind) + # A tree report groups results by their execution identity, regardless of + # whether the result came from a build, boot, or test endpoint. + identity.pop("kind") + return tuple(identity.values()) + + +def _duplicate_sort_key(item): + """Order otherwise-identical results before pairing them across revisions.""" + return ( + str(item.get("status") or "UNKNOWN").upper(), + str(item.get("id") or ""), + ) + + +def _tree_report_keys(tree_report, category): + keys = set() + for platform, configs in (tree_report or {}).get(category, {}).items(): + for config, arch_compilers in configs.items(): + for arch_compiler, paths in arch_compilers.items(): + architecture, _, compiler = arch_compiler.partition("/") + for path, tests in paths.items(): + for test in tests or [{}]: + identity = result_identity( + { + **test, + "origin": (tree_report or {}).get("origin"), + "platform": platform, + "config": config, + "architecture": architecture, + "compiler": compiler, + "path": path, + }, + "test", + ) + identity.pop("kind") + keys.add(tuple(identity.values())) + return keys + + +@dataclass +class RegressionReport: + """A deterministic comparison report which preserves duplicate results.""" + + base: str + head: str + items: list = field(default_factory=list) + incomplete: bool = False + + @classmethod + def compare(cls, base, head, base_results, head_results, tree_report=None): + report = cls(base=base, head=head) + history = { + "regression": _tree_report_keys(tree_report, "possible_regressions"), + "fixed": _tree_report_keys(tree_report, "fixed_regressions"), + "unstable": _tree_report_keys(tree_report, "unstable_tests"), + } + for kind in ("build", "boot", "test"): + old = defaultdict(list) + new = defaultdict(list) + for item in base_results.get(kind + "s", []): + old[_key(item, kind)].append(item) + for item in head_results.get(kind + "s", []): + new[_key(item, kind)].append(item) + for key in sorted(set(old) | set(new)): + # The API does not guarantee result order. + old[key].sort(key=_duplicate_sort_key) + new[key].sort(key=_duplicate_sort_key) + count = max(len(old[key]), len(new[key])) + for occurrence in range(count): + before = ( + old[key][occurrence] if occurrence < len(old[key]) else None + ) + after = new[key][occurrence] if occurrence < len(new[key]) else None + category = cls._classify(before, after, key, history) + if category is None: + continue + chosen = after if after is not None else before + report.items.append( + { + "category": category, + "identity": result_identity(chosen, kind), + "occurrence": occurrence, + "base_status": before.get("status") if before else None, + "head_status": after.get("status") if after else None, + "base_id": before.get("id") if before else None, + "head_id": after.get("id") if after else None, + "known_issues": [], + } + ) + return report + + @staticmethod + def _classify(before, after, key, history): + if before is None: + return "new" + if after is None: + return "missing" + old = str(before.get("status") or "UNKNOWN").upper() + new = str(after.get("status") or "UNKNOWN").upper() + if key in history["unstable"]: + return "unstable" + if new in FAIL_STATUSES and (key in history["regression"] or old == "PASS"): + return "regression" + if new == "PASS" and (key in history["fixed"] or old in FAIL_STATUSES): + return "fixed" + if old in FAIL_STATUSES and new in FAIL_STATUSES: + return "persistent_fail" + if old != new: + return "unstable" + return None + + @property + def counts(self): + counts = Counter(item["category"] for item in self.items) + return {category: counts[category] for category in CATEGORIES} + + def has_violation(self, fail_on="regression"): + policies = {part.strip() for part in fail_on.split(",") if part.strip()} + return any(self.counts.get(policy, 0) for policy in policies) + + def to_dict(self): + return { + "base": self.base, + "head": self.head, + "counts": self.counts, + "incomplete": self.incomplete, + "items": self.items, + } diff --git a/kcidev/mcp/tools_dashboard.py b/kcidev/mcp/tools_dashboard.py index 6f15c03..44fa05f 100644 --- a/kcidev/mcp/tools_dashboard.py +++ b/kcidev/mcp/tools_dashboard.py @@ -78,6 +78,26 @@ def get_summary( } +@tool_errors +def compare_checkouts( + giturl: str, + branch: str, + base: str, + head: str, + origin: str = "maestro", + include_issues: bool = False, +): + """Compare two checkouts, classifying regressions, fixes and unstable tests. + The returned report preserves duplicate executions. Set ``include_issues`` + to look up known issue ids for failing/regressing results; this can require + one additional request per result. ``incomplete`` means the report must not + be treated as a successful CI gate. + """ + return _current_client().compare_results( + base, head, giturl, branch, origin, include_issues=include_issues + ) + + @tool_errors def list_commits(giturl: str, branch: str, commit: str, origin: str = "maestro"): """List recent checkouts of a tree with per-commit result counts. @@ -294,6 +314,7 @@ def get_issue_tests( READ_ONLY_TOOLS = ( list_trees, get_summary, + compare_checkouts, list_commits, list_builds, list_boots, diff --git a/kcidev/subcommands/results/__init__.py b/kcidev/subcommands/results/__init__.py index 20965a2..70fdb4f 100644 --- a/kcidev/subcommands/results/__init__.py +++ b/kcidev/subcommands/results/__init__.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- +import json import sys from functools import wraps @@ -25,6 +26,7 @@ dashboard_fetch_tree_report, ) from kcidev.libs.git_repo import get_tree_name, set_giturl_branch_commit +from kcidev.libs.regression import CATEGORIES from kcidev.subcommands.results.hardware import hardware from kcidev.subcommands.results.options import ( builds_and_tests_options, @@ -384,8 +386,18 @@ def boot(op_id, download_logs, use_json): default=True, ) @click.argument("commits", nargs=-1, required=False) -@results_display_options -def compare(origin, giturl, branch, latest, commits, use_json): +@click.option( + "--format", "output_format", type=click.Choice(["human", "json"]), default="human" +) +@click.option("--json", "use_json", is_flag=True, hidden=True) +@click.option( + "--include-issues", + is_flag=True, + help="Look up known issues (one request per regression/persistent failure)", +) +def compare( + origin, giturl, branch, latest, commits, output_format, use_json, include_issues +): """Compare test results between commits with summary and regressions. Compares test results between commits showing summary statistics @@ -404,15 +416,94 @@ def compare(origin, giturl, branch, latest, commits, use_json): # Compare specific commits kci-dev results compare --giturl https://git.kernel.org/... abc123 def456 """ - if latest and not commits: - # Use latest commits from history - cmd_compare(origin, giturl, branch, None, use_json) - elif len(commits) == 2: - # Use specific commits provided - cmd_compare(origin, giturl, branch, list(commits), use_json) + from kcidev.api import KciDevError, KernelCIClient + + json_output = use_json or output_format == "json" + if len(commits) != 2: + raise click.UsageError("exactly BASE and HEAD commits are required") + try: + report = KernelCIClient().compare_results( + commits[0], + commits[1], + giturl, + branch, + origin, + include_issues=include_issues, + ) + except KciDevError as exc: + if json_output: + click.echo(json.dumps({"error": str(exc), "incomplete": True})) + raise click.exceptions.Exit(2) from exc + if json_output: + # This is deliberately the only stdout write in JSON mode. + click.echo(json.dumps(report, sort_keys=True)) + else: - click.echo("Error: Provide either --latest flag or exactly 2 commit hashes") - raise click.Abort() + click.echo(f"Compared {commits[0]} -> {commits[1]}") + for category, count in report["counts"].items(): + click.echo(f" {category}: {count}") + if report["incomplete"]: + raise click.exceptions.Exit(2) + if report["counts"]["regression"]: + raise click.exceptions.Exit(1) + + +@results.command() +@click.option("--origin", default="maestro", help="Select KCIDB origin") +@click.option("--giturl", required=True, help="Git repository URL") +@click.option("--branch", required=True, help="Git branch name") +@click.option("--base", help="Baseline checkout commit (defaults to previous)") +@click.option("--head", help="Candidate checkout commit (defaults to latest)") +@click.option( + "--fail-on", + default="regression", + show_default=True, + help="Comma-separated report categories which fail the gate", +) +@click.option( + "--format", "output_format", type=click.Choice(["human", "json"]), default="human" +) +def gate(origin, giturl, branch, base, head, fail_on, output_format): + """Gate a checkout using the regression comparison policy.""" + from kcidev.api import KciDevError, KernelCIClient + + policies = {value.strip() for value in fail_on.split(",") if value.strip()} + unknown_policies = policies.difference(CATEGORIES) + if unknown_policies: + unknown = ", ".join(sorted(unknown_policies)) + raise click.UsageError(f"unknown --fail-on categories: {unknown}") + + try: + client = KernelCIClient() + if bool(base) != bool(head): + raise click.UsageError("provide both --base and --head, or neither") + if not base: + giturl, branch, latest = set_giturl_branch_commit( + origin, giturl, branch, None, True, None + ) + history = client.get_commits_history(origin, giturl, branch, latest) + commits = ( + history if isinstance(history, list) else history.get("commits", []) + ) + if len(commits) < 2: + raise KciDevError("fewer than two checkouts are available") + head, base = commits[0]["git_commit_hash"], commits[1]["git_commit_hash"] + report = client.compare_results(base, head, giturl, branch, origin) + except (KciDevError, click.Abort) as exc: + if output_format == "json": + click.echo(json.dumps({"error": str(exc), "incomplete": True})) + else: + click.echo(f"Incomplete comparison: {exc}", err=True) + raise click.exceptions.Exit(2) from exc + click.echo( + json.dumps(report, sort_keys=True) + if output_format == "json" + else "\n".join(f"{key}: {value}" for key, value in report["counts"].items()) + ) + if report["incomplete"]: + raise click.exceptions.Exit(2) + if any(report["counts"].get(value, 0) for value in policies): + raise click.exceptions.Exit(1) def get_issues(ctx, origin, item_type, giturl, branch, commit, tree_name, arch): diff --git a/poetry.toml b/poetry.toml new file mode 100644 index 0000000..6742b30 --- /dev/null +++ b/poetry.toml @@ -0,0 +1,2 @@ +[installer] +parallel = false diff --git a/tests/test_api.py b/tests/test_api.py index 19eae86..000451e 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -222,3 +222,82 @@ def test_trigger_patchset_failure_raises(monkeypatch): monkeypatch.setattr(maestro_common.kcidev_session, "post", post) with pytest.raises(KciDevError, match="patchset failed"): _client().trigger_patchset("0" * 24, patches=["patch content"]) + + +def test_get_tree_report_uses_integer_age_defaults(monkeypatch): + client = _client() + request = Mock(return_value={}) + monkeypatch.setattr(client, "_dashboard_request", request) + + client.get_tree_report("maestro", "main", "https://example.com/linux.git") + + assert request.call_args.args[-2:] == (24, 0) + + +@pytest.mark.parametrize( + ("method_name", "result_id"), + [("get_build_issues", "build-id"), ("get_boot_issues", "test-id")], +) +def test_get_issues_forwards_error_verbosity(monkeypatch, method_name, result_id): + client = _client() + request = Mock(return_value=[]) + monkeypatch.setattr(client, "_dashboard_request", request) + + getattr(client, method_name)(result_id, error_verbose=False) + + assert request.call_args.args[-2:] == (True, False) + + +def test_compare_results_continues_without_tree_history(monkeypatch): + client = _client() + monkeypatch.setattr(client, "get_builds", Mock(return_value={"builds": []})) + monkeypatch.setattr(client, "get_boots", Mock(return_value={"boots": []})) + monkeypatch.setattr(client, "get_tests", Mock(return_value={"tests": []})) + monkeypatch.setattr( + client, + "get_tree_report", + Mock(side_effect=KciDevError("tree report unavailable")), + ) + + report = client.compare_results( + "base", "head", "https://example.com/linux.git", "main" + ) + + assert report["incomplete"] is True + assert report["items"] == [] + + +def test_compare_results_only_fetches_issues_when_requested(monkeypatch): + client = _client() + passing = { + "id": "base-test", + "status": "PASS", + "path": "suite.case", + "environment_misc": {"platform": "qemu"}, + } + failing = {**passing, "id": "head-test", "status": "FAIL"} + monkeypatch.setattr(client, "get_builds", Mock(return_value={"builds": []})) + monkeypatch.setattr(client, "get_boots", Mock(return_value={"boots": []})) + get_tests = Mock(side_effect=[{"tests": [passing]}, {"tests": [failing]}] * 2) + monkeypatch.setattr(client, "get_tests", get_tests) + monkeypatch.setattr(client, "get_tree_report", Mock(return_value={})) + get_issues = Mock(return_value=[{"id": "issue-1"}]) + monkeypatch.setattr(client, "get_boot_issues", get_issues) + + report = client.compare_results( + "base", "head", "https://example.com/linux.git", "main" + ) + + assert report["items"][0]["known_issues"] == [] + get_issues.assert_not_called() + + report = client.compare_results( + "base", + "head", + "https://example.com/linux.git", + "main", + include_issues=True, + ) + + assert report["items"][0]["known_issues"] == ["issue-1"] + get_issues.assert_called_once_with("head-test", error_verbose=False) diff --git a/tests/test_regression.py b/tests/test_regression.py new file mode 100644 index 0000000..c67cfe2 --- /dev/null +++ b/tests/test_regression.py @@ -0,0 +1,546 @@ +import json + +import click +from click.testing import CliRunner + +from kcidev.libs.regression import RegressionReport +from kcidev.main import get_cli + + +def result(result_id, status, path="suite.case", platform="qemu"): + return { + "id": result_id, + "origin": "maestro", + "status": status, + "path": path, + "environment_misc": {"platform": platform}, + "architecture": "arm64", + "compiler": "gcc-14", + "config": "defconfig", + } + + +def test_report_classifies_transitions_and_preserves_duplicates(): + base = { + "tests": [ + result("r1", "PASS"), + result("r2", "PASS"), + result("f", "FAIL", "fixed"), + result("p", "FAIL", "persistent"), + result("m", "PASS", "missing"), + result("u", "SKIP", "unstable"), + ] + } + head = { + "tests": [ + result("r3", "FAIL"), + result("r4", "FAIL"), + result("f2", "PASS", "fixed"), + result("p2", "FAIL", "persistent"), + result("n", "PASS", "new"), + result("u2", "PASS", "unstable"), + ] + } + report = RegressionReport.compare("base", "head", base, head) + assert report.counts == { + "regression": 2, + "fixed": 1, + "unstable": 1, + "persistent_fail": 1, + "new": 1, + "missing": 1, + } + regressions = [i for i in report.items if i["category"] == "regression"] + assert [i["occurrence"] for i in regressions] == [0, 1] + assert regressions[0]["identity"] == { + "kind": "test", + "origin": "maestro", + "platform": "qemu", + "architecture": "arm64", + "compiler": "gcc-14", + "config": "defconfig", + "path": "suite.case", + } + + +def test_report_preserves_empty_head_result(): + report = RegressionReport.compare("base", "head", {}, {"tests": [{}]}) + + assert report.items == [ + { + "category": "new", + "identity": { + "kind": "test", + "origin": "unknown", + "platform": "unknown", + "architecture": "unknown", + "compiler": "unknown", + "config": "unknown", + "path": "unknown", + }, + "occurrence": 0, + "base_status": None, + "head_status": None, + "base_id": None, + "head_id": None, + "known_issues": [], + } + ] + + +def test_report_pairs_duplicate_results_independently_of_api_order(): + base_results = [result("base-pass", "PASS"), result("base-fail", "FAIL")] + head_results = [result("head-fail", "FAIL"), result("head-pass", "PASS")] + + report = RegressionReport.compare( + "base", "head", {"tests": base_results}, {"tests": head_results} + ) + reversed_report = RegressionReport.compare( + "base", + "head", + {"tests": list(reversed(base_results))}, + {"tests": list(reversed(head_results))}, + ) + + assert report.to_dict() == reversed_report.to_dict() + assert report.counts == { + "regression": 0, + "fixed": 0, + "unstable": 0, + "persistent_fail": 1, + "new": 0, + "missing": 0, + } + assert report.items[0]["base_id"] == "base-fail" + assert report.items[0]["head_id"] == "head-fail" + + +def test_report_marks_unpaired_duplicate_as_missing(): + report = RegressionReport.compare( + "base", + "head", + { + "tests": [ + result("base-first", "PASS"), + result("base-second", "PASS"), + ] + }, + {"tests": [result("head-only", "PASS")]}, + ) + + assert report.counts == { + "regression": 0, + "fixed": 0, + "unstable": 0, + "persistent_fail": 0, + "new": 0, + "missing": 1, + } + assert report.items == [ + { + "category": "missing", + "identity": { + "kind": "test", + "origin": "maestro", + "platform": "qemu", + "architecture": "arm64", + "compiler": "gcc-14", + "config": "defconfig", + "path": "suite.case", + }, + "occurrence": 1, + "base_status": "PASS", + "head_status": None, + "base_id": "base-second", + "head_id": None, + "known_issues": [], + } + ] + + +def test_history_categories_only_apply_to_matching_head_status(): + key = ("maestro", "qemu", "arm64", "gcc-14", "defconfig", "suite.case") + regression_history = {"regression": {key}, "fixed": set(), "unstable": set()} + assert ( + RegressionReport._classify( + result("old", "PASS"), result("new", "PASS"), key, regression_history + ) + is None + ) + assert ( + RegressionReport._classify( + result("old", "FAIL"), result("new", "PASS"), key, regression_history + ) + == "fixed" + ) + assert ( + RegressionReport._classify( + result("old", "PASS"), result("new", "FAIL"), key, regression_history + ) + == "regression" + ) + assert ( + RegressionReport._classify( + result("old", "FAIL"), result("new", "FAIL"), key, regression_history + ) + == "regression" + ) + + fixed_history = {"regression": set(), "fixed": {key}, "unstable": set()} + assert ( + RegressionReport._classify( + result("old", "PASS"), result("new", "FAIL"), key, fixed_history + ) + == "regression" + ) + assert ( + RegressionReport._classify( + result("old", "FAIL"), result("new", "FAIL"), key, fixed_history + ) + == "persistent_fail" + ) + assert ( + RegressionReport._classify( + result("old", "PASS"), result("new", "PASS"), key, fixed_history + ) + == "fixed" + ) + + unstable_history = {"regression": set(), "fixed": set(), "unstable": {key}} + assert ( + RegressionReport._classify( + result("old", "PASS"), result("new", "PASS"), key, unstable_history + ) + == "unstable" + ) + + +def test_tree_history_uses_report_origin_and_matches_all_result_kinds(): + tree_report = { + "origin": "maestro", + "possible_regressions": { + "qemu": { + "defconfig": { + "arm64/gcc-14": { + "build": [{}], + "boot": [{}], + "suite.case": [{}], + } + } + } + }, + } + base = { + "builds": [result("build-old", "FAIL", path=None)], + "boots": [result("boot-old", "FAIL", path="boot")], + "tests": [result("test-old", "FAIL")], + } + head = { + "builds": [result("build-new", "FAIL", path=None)], + "boots": [result("boot-new", "FAIL", path="boot")], + "tests": [result("test-new", "FAIL")], + } + + report = RegressionReport.compare("base", "head", base, head, tree_report) + + assert report.counts["regression"] == 3 + assert {item["identity"]["kind"] for item in report.items} == { + "build", + "boot", + "test", + } + + +def test_compare_json_is_one_document_and_regressions_exit_one(monkeypatch): + report = RegressionReport("base", "head") + report.items.append({"category": "regression"}) + monkeypatch.setattr( + "kcidev.api.KernelCIClient.compare_results", + lambda *args, **kwargs: report.to_dict(), + ) + result = CliRunner().invoke( + get_cli(), + [ + "results", + "compare", + "--giturl", + "url", + "--branch", + "main", + "--format", + "json", + "base", + "head", + ], + ) + assert result.exit_code == 1 + assert json.loads(result.stdout)["counts"]["regression"] == 1 + + +def test_gate_rejects_unknown_fail_on_categories_before_comparison(monkeypatch): + def unexpected_comparison(*args, **kwargs): + raise AssertionError("comparison should not run for an invalid policy") + + monkeypatch.setattr( + "kcidev.api.KernelCIClient.compare_results", unexpected_comparison + ) + result = CliRunner().invoke( + get_cli(), + [ + "results", + "gate", + "--giturl", + "url", + "--branch", + "main", + "--base", + "base", + "--head", + "head", + "--fail-on", + "unknown, regression, typo", + ], + ) + + assert result.exit_code == 2 + assert "unknown --fail-on categories: typo, unknown" in result.output + + +def test_gate_handles_abort_while_resolving_latest_checkout(monkeypatch): + def abort_resolution(*args, **kwargs): + raise click.Abort() + + monkeypatch.setattr( + "kcidev.subcommands.results.set_giturl_branch_commit", abort_resolution + ) + result = CliRunner().invoke( + get_cli(), + [ + "results", + "gate", + "--giturl", + "url", + "--branch", + "main", + "--format", + "json", + ], + ) + + assert result.exit_code == 2 + assert json.loads(result.stdout) == {"error": "", "incomplete": True} + + +def _report(**counts): + return { + "base": "base", + "head": "head", + "counts": { + category: counts.get(category, 0) + for category in ( + "regression", + "fixed", + "unstable", + "persistent_fail", + "new", + "missing", + ) + }, + "incomplete": counts.get("incomplete", False), + "items": [], + } + + +def test_gate_parses_fail_on_list_and_exits_for_selected_category(monkeypatch): + monkeypatch.setattr( + "kcidev.api.KernelCIClient.compare_results", + lambda *args, **kwargs: _report(fixed=1), + ) + + result = CliRunner().invoke( + get_cli(), + [ + "results", + "gate", + "--giturl", + "url", + "--branch", + "main", + "--base", + "base", + "--head", + "head", + "--fail-on", + " regression, , fixed,regression ", + ], + ) + + assert result.exit_code == 1 + assert "fixed: 1" in result.stdout + + +def test_gate_exits_zero_when_policy_has_no_violations(monkeypatch): + monkeypatch.setattr( + "kcidev.api.KernelCIClient.compare_results", + lambda *args, **kwargs: _report(fixed=1), + ) + + result = CliRunner().invoke( + get_cli(), + [ + "results", + "gate", + "--giturl", + "url", + "--branch", + "main", + "--base", + "base", + "--head", + "head", + "--fail-on", + "regression", + "--format", + "json", + ], + ) + + assert result.exit_code == 0 + assert json.loads(result.stdout)["counts"]["fixed"] == 1 + + +def test_gate_exits_two_for_incomplete_report_before_policy(monkeypatch): + monkeypatch.setattr( + "kcidev.api.KernelCIClient.compare_results", + lambda *args, **kwargs: _report(regression=1, incomplete=True), + ) + + result = CliRunner().invoke( + get_cli(), + [ + "results", + "gate", + "--giturl", + "url", + "--branch", + "main", + "--base", + "base", + "--head", + "head", + ], + ) + + assert result.exit_code == 2 + + +def test_gate_requires_base_and_head_together(): + result = CliRunner().invoke( + get_cli(), + [ + "results", + "gate", + "--giturl", + "url", + "--branch", + "main", + "--base", + "base", + ], + ) + + assert result.exit_code == 2 + assert "provide both --base and --head, or neither" in result.output + + +def test_gate_resolves_latest_pair_from_history_dictionary(monkeypatch): + monkeypatch.setattr( + "kcidev.subcommands.results.set_giturl_branch_commit", + lambda *args, **kwargs: ("resolved-url", "resolved-branch", "latest"), + ) + monkeypatch.setattr( + "kcidev.api.KernelCIClient.get_commits_history", + lambda *args, **kwargs: { + "commits": [ + {"git_commit_hash": "head-from-history"}, + {"git_commit_hash": "base-from-history"}, + ] + }, + ) + compared = {} + + def compare(_client, base, head, giturl, branch, origin): + compared["args"] = (base, head, giturl, branch, origin) + return _report() + + monkeypatch.setattr("kcidev.api.KernelCIClient.compare_results", compare) + + result = CliRunner().invoke( + get_cli(), + ["results", "gate", "--giturl", "url", "--branch", "main"], + ) + + assert result.exit_code == 0 + assert compared["args"] == ( + "base-from-history", + "head-from-history", + "resolved-url", + "resolved-branch", + "maestro", + ) + + +def test_gate_accepts_list_history(monkeypatch): + monkeypatch.setattr( + "kcidev.subcommands.results.set_giturl_branch_commit", + lambda *args, **kwargs: ("url", "main", "latest"), + ) + monkeypatch.setattr( + "kcidev.api.KernelCIClient.get_commits_history", + lambda *args, **kwargs: [ + {"git_commit_hash": "head"}, + {"git_commit_hash": "base"}, + ], + ) + monkeypatch.setattr( + "kcidev.api.KernelCIClient.compare_results", + lambda *args, **kwargs: _report(), + ) + + result = CliRunner().invoke( + get_cli(), + ["results", "gate", "--giturl", "url", "--branch", "main"], + ) + + assert result.exit_code == 0 + + +def test_gate_exits_two_when_history_has_fewer_than_two_commits(monkeypatch): + monkeypatch.setattr( + "kcidev.subcommands.results.set_giturl_branch_commit", + lambda *args, **kwargs: ("url", "main", "latest"), + ) + monkeypatch.setattr( + "kcidev.api.KernelCIClient.get_commits_history", + lambda *args, **kwargs: {"commits": [{"git_commit_hash": "only"}]}, + ) + + result = CliRunner().invoke( + get_cli(), + [ + "results", + "gate", + "--giturl", + "url", + "--branch", + "main", + "--format", + "json", + ], + ) + + assert result.exit_code == 2 + assert json.loads(result.stdout) == { + "error": "fewer than two checkouts are available", + "incomplete": True, + }