diff --git a/src/transformersci/agentic/integration_failure_triage.py b/src/transformersci/agentic/integration_failure_triage.py index 38ca00a..29e3d71 100644 --- a/src/transformersci/agentic/integration_failure_triage.py +++ b/src/transformersci/agentic/integration_failure_triage.py @@ -569,7 +569,14 @@ def oom_shapes(target: dict) -> dict[str, tuple[str, dict[str, float]]]: _E_EXC_PAT = re.compile( r"^\s*E\s+(?:[\w.]+\.)?([A-Za-z_]\w*(?:Error|Exception)):?(.*)$" ) -_BARE_EXC_PAT = re.compile(r"^(?:[\w.]+\.)?([A-Za-z_]\w*(?:Error|Exception)):?(.*)$") +# The CI dataset's ``model_results.json`` writes each failure as +# ``(line 130) AssertionError: …`` — the failing line, then the exception. Until +# 2026-10-11 that prefix defeated this pattern on every trace, so every group's +# terminal exception came out "other" and assertions were classified by symptom +# text alone (86 of 222 on 2026-10-10 ended up `other`). +_BARE_EXC_PAT = re.compile( + r"^(?:\(line \d+\)\s+)?(?:[\w.]+\.)?([A-Za-z_]\w*(?:Error|Exception)):?(.*)$" +) # The location pytest prints for the raising frame: ``path/to/file.py:123: ExcType``. _CRASH_SITE_PAT = re.compile( r"^([\w./-]+\.py):(\d+):\s+[A-Za-z_]\w*(?:Error|Exception)\b" @@ -1226,32 +1233,6 @@ def pick_targets(report: dict) -> list[dict]: return targets -# The CI dataset writes each failure as ``(line 130) AssertionError: …``, a -# form :func:`terminal_exception` does not parse, so in production every group's -# ``terminal_exc`` is "other" (all 376 integration failures on 2026-10-10, -# 222 of them AssertionErrors). The two routing decisions below that ask "is -# this really an assertion?" read the exception from that prefix instead. The -# grouping and the fingerprint still use ``terminal_exc``; changing those would -# re-key every open group, so it is a separate change. -_LINE_PREFIXED_EXC = re.compile( - r"^\(line \d+\)\s+(?:[\w.]+\.)?([A-Za-z_]\w*(?:Error|Exception))\b" -) - - -def routing_exc(target: dict) -> str: - """The group's terminal exception for routing: ``terminal_exc`` when it was - parsed, else the exception every failure's ``(line N)`` trace names.""" - exc = target.get("terminal_exc") or "" - if exc not in ("", "other"): - return exc - seen = set() - for f in target.get("failures") or []: - trace = (f.get("latest_trace") or f.get("trace") or "").strip() - m = _LINE_PREFIXED_EXC.match(trace) - seen.add(m.group(1) if m else None) - return seen.pop() or exc if len(seen) == 1 else exc - - def dispatch_category(target: dict) -> str: """The category a group is really in, for selection diversity. @@ -1265,7 +1246,7 @@ def dispatch_category(target: dict) -> str: if target.get("kind") != "model_failures": return "cluster" mode = target.get("failure_mode") or "other" - if mode in _CRASH_MODES and routing_exc(target) == "AssertionError": + if mode in _CRASH_MODES and target.get("terminal_exc") == "AssertionError": return "output_mismatch" return mode @@ -2347,6 +2328,15 @@ def target_tests(target: dict) -> set[str]: return {f["test"] for f in target.get("failures") or [] if f.get("test")} +# Fixing `_BARE_EXC_PAT` for the CI dataset's "(line N)" traces (2026-10-12) +# reclassified about a fifth of the groups (other → output_mismatch), which +# changed their labels and so their fingerprints: 68 of 110 on the 2026-10-11 +# window. A closed PR opened before that date can only be found by the tests it +# lists, so it matches by test overlap, which keeps its rejection and the +# maintainer feedback attached to it (big_bird's #48437, "do not record "). +FINGERPRINT_REKEY_DATE = "2026-10-12" + + def _pr_matches(pr: dict, markers: list[str], prefixes: list[str]) -> bool: body = pr.get("body") or "" head_ref = (pr.get("head") or {}).get("ref") or "" @@ -2371,8 +2361,9 @@ def classify_prior_attempts( tests…", a test flaking on one runner). Each move minted a new fingerprint and so a second PR for the same test: on 2026-10-11 six failure groups had two open serge PRs each. So an open ITF PR whose body lists any of the group's tests is - its existing PR too, and the group gets a follow-up on it. Closed PRs still - match by fingerprint only, so the rejection count keeps its meaning. + its existing PR too, and the group gets a follow-up on it. Closed PRs match + by fingerprint only, so the rejection count keeps its meaning, except those + opened before :data:`FINGERPRINT_REKEY_DATE`, which also match by test. A PR serge closed at this triage's request (:data:`SERGE_CLOSE_MARKERS`) is ignored: retiring a stale or duplicate PR is not a maintainer rejecting a @@ -2384,8 +2375,12 @@ def classify_prior_attempts( rejected: list[int] = [] for pr in pulls: is_open = (pr.get("state") or "") != "closed" + pre_rekey = (pr.get("created_at") or "9999") < FINGERPRINT_REKEY_DATE if not _pr_matches(pr, markers, prefixes) and not ( - tests and is_open and is_itf_pr(pr) and tests & pr_target_tests(pr) + tests + and (is_open or pre_rekey) + and is_itf_pr(pr) + and tests & pr_target_tests(pr) ): continue if not is_open and serge_closed_reason(pr): @@ -4673,7 +4668,7 @@ def instruction_addendum(target: dict) -> str: if mode in _CRASH_MODES: # An `AssertionError` reaching us as `other` is still an assertion, so it # gets the mismatch guidance rather than "this is a library bug". - if routing_exc(target) == "AssertionError": + if target.get("terminal_exc") == "AssertionError": return _MISMATCH_GUIDANCE # A crash raised inside the weight-conversion step is a loading/placement # problem, not a library bug to hunt — see `_CONVERSION_PAT`. Checked only diff --git a/tests/test_integration_failure_triage.py b/tests/test_integration_failure_triage.py index c72f73d..bea92b3 100644 --- a/tests/test_integration_failure_triage.py +++ b/tests/test_integration_failure_triage.py @@ -156,6 +156,28 @@ def test_assertion_error(self): def test_none_when_no_exception(self): self.assertEqual(itf.terminal_exception("just a plain line"), (None, "")) + def test_ci_dataset_line_prefix(self): + """Every trace in the CI dataset's model_results.json reads + "(line N) : msg" (all 376 integration failures on 2026-10-10).""" + self.assertEqual( + itf.terminal_exception("(line 130) AssertionError: 'a' != 'b'"), + ("AssertionError", "'a' != 'b'"), + ) + self.assertEqual( + itf.terminal_exception( + "(line 88) torch.OutOfMemoryError: CUDA out of memory" + ), + ("OutOfMemoryError", "CUDA out of memory"), + ) + + def test_line_prefixed_assertion_is_an_output_mismatch(self): + # 86 of the 222 AssertionErrors on 2026-10-10 classified `other`, so + # those groups were sent the crash guidance. + self.assertEqual( + itf.classify("(line 130) AssertionError: 'x' != 'y'"), "output_mismatch" + ) + self.assertEqual(itf.classify("(line 5) ValueError: bad shape"), "other") + def test_crash_site_is_raising_frame(self): self.assertEqual( itf.crash_site(_TRACE_UNBOUND_CRASH), @@ -553,6 +575,24 @@ def test_test_overlap_never_creates_a_rejection(self): prior = itf.classify_prior_attempts([pr], [self.FP], {t}) self.assertEqual(prior, itf.PriorAttempts()) + def test_pre_rekey_closed_pr_keeps_its_rejection_by_test(self): + """The 2026-10-12 parser fix changed most fingerprints; a PR closed + before it can only be found by its tests (big_bird #48437).""" + t = "tests/models/x/test_modeling_x.py::XIntegrationTest::test_a" + old = { + "number": 48437, + "state": "closed", + "created_at": "2026-08-31T10:00:00Z", + "body": f"- `{t}` [single-gpu]", + "head": {"ref": "serge/fix/itf-aaaaaaaaaaaa-1"}, + } + prior = itf.classify_prior_attempts([old], [self.FP], {t}) + self.assertEqual(prior.rejected, (48437,)) + newer = {**old, "number": 50000, "created_at": "2026-10-20T10:00:00Z"} + self.assertEqual( + itf.classify_prior_attempts([newer], [self.FP], {t}).rejected, () + ) + def test_test_overlap_ignores_non_itf_branches(self): t = "tests/models/x/test_modeling_x.py::XIntegrationTest::test_a" pr = {"number": 5, "state": "open", "body": f"`{t}`", "head": {"ref": "fix-x"}} @@ -2835,29 +2875,18 @@ def test_mismatch_names_the_runner_key_and_the_protected_ones(self): self.assertIn(protected, text) def test_line_prefixed_assertion_gets_the_mismatch_guidance(self): - """The CI dataset writes "(line 130) AssertionError: ...", which - terminal_exception() does not parse, so real groups carry - terminal_exc "other" (transformers#49336). Before routing_exc they got - the CRASH block and no rule about which key to write.""" - target = _target("other", exc="other") - target["failures"][0]["latest_trace"] = ( - "(line 130) AssertionError: 'My favourite condiment is' != 'olive oil'" - ) - self.assertEqual(itf.routing_exc(target), "AssertionError") + """The CI dataset writes "(line 130) AssertionError: ...". Before + 2026-10-12 terminal_exception() could not parse that, so real groups + carried terminal_exc "other" and got the CRASH block (transformers#49336). + Built through cluster_failures/pick_targets, as the nightly builds it.""" + trace = "(line 130) AssertionError: 'My favourite condiment is' != 'olive oil'" + f = _failure("ministral3", "multi", "tests/models/ministral3/t.py::T::t", trace) + f.pop("failure_mode", None) + targets = itf.pick_targets(itf.cluster_failures([f], None, attribution={})) + target = next(t for t in targets if t.get("kind") == "model_failures") + self.assertEqual(target["terminal_exc"], "AssertionError") + self.assertEqual(target["failure_mode"], "output_mismatch") self.assertEqual(itf.instruction_addendum(target), itf._MISMATCH_GUIDANCE) - self.assertEqual(itf.dispatch_category(target), "output_mismatch") - - def test_line_prefixed_crash_keeps_the_crash_guidance(self): - target = _target("other", exc="other") - target["failures"][0]["latest_trace"] = "(line 88) ValueError: bad shape" - self.assertEqual(itf.routing_exc(target), "ValueError") - self.assertNotIn("#48198", itf.instruction_addendum(target)) - - def test_mixed_exceptions_do_not_route_as_an_assertion(self): - target = _target("other", exc="other", n=2) - target["failures"][0]["latest_trace"] = "(line 1) AssertionError: x" - target["failures"][1]["latest_trace"] = "(line 2) ValueError: y" - self.assertEqual(itf.routing_exc(target), "other") def test_the_convention_stays_out_of_a_crash_group(self): """A crash never got as far as comparing values, so "add a device key" diff --git a/utils/prompt_ab.py b/utils/prompt_ab.py index de0aee3..7d1e084 100644 --- a/utils/prompt_ab.py +++ b/utils/prompt_ab.py @@ -114,16 +114,16 @@ } ], }, - # transformers#49336, as the triage really builds it: the CI dataset's - # "(line N) AssertionError" trace leaves terminal_exc "other", which before - # routing_exc sent this expectation group the CRASH guidance ("do not edit - # the test") and so no rule on which Expectations key to write. + # transformers#49336, as the triage builds it since 2026-10-12. Before that, + # the CI dataset's "(line N) AssertionError" trace left terminal_exc + # "other" and the mode `other`, so this expectation group got the CRASH + # guidance ("do not edit the test") and no rule on which key to write. "ministral3-line-prefixed-assertion": { "kind": "model_failures", - "label": "2 integration tests for model `ministral3` failing with `other`", + "label": "2 integration tests for model `ministral3` failing with `output_mismatch`", "model": "ministral3", - "failure_mode": "other", - "terminal_exc": "other", + "failure_mode": "output_mismatch", + "terminal_exc": "AssertionError", "failures": [ { "test": ( @@ -131,7 +131,7 @@ "::Ministral3IntegrationTest::test_model_3b_generation" ), "gpu": "multi-gpu", - "failure_mode": "other", + "failure_mode": "output_mismatch", "latest_trace": ( "(line 130) AssertionError: 'My favourite condiment is 100% " "pure oliv[64 chars] can' != \"My favourite condiment is 100% "