ITF: parse the CI dataset's "(line N)" traces

src/transformersci/agentic/integration_failure_triage.py

@@ -569,7 +569,14 @@ def oom_shapes(target: dict) -> dict[str, tuple[str, dict[str, float]]]:
 _E_EXC_PAT = re.compile(
     r"^\s*E\s+(?:[\w.]+\.)?([A-Za-z_]\w*(?:Error|Exception)):?(.*)$"
 )
-_BARE_EXC_PAT = re.compile(r"^(?:[\w.]+\.)?([A-Za-z_]\w*(?:Error|Exception)):?(.*)$")
+# The CI dataset's ``model_results.json`` writes each failure as
+# ``(line 130)  AssertionError: …`` — the failing line, then the exception. Until
+# 2026-10-11 that prefix defeated this pattern on every trace, so every group's
+# terminal exception came out "other" and assertions were classified by symptom
+# text alone (86 of 222 on 2026-10-10 ended up `other`).
+_BARE_EXC_PAT = re.compile(
+    r"^(?:\(line \d+\)\s+)?(?:[\w.]+\.)?([A-Za-z_]\w*(?:Error|Exception)):?(.*)$"
+)
 # The location pytest prints for the raising frame: ``path/to/file.py:123: ExcType``.
 _CRASH_SITE_PAT = re.compile(
     r"^([\w./-]+\.py):(\d+):\s+[A-Za-z_]\w*(?:Error|Exception)\b"
@@ -1226,32 +1233,6 @@ def pick_targets(report: dict) -> list[dict]:
     return targets
 
 
-# The CI dataset writes each failure as ``(line 130)  AssertionError: …``, a
-# form :func:`terminal_exception` does not parse, so in production every group's
-# ``terminal_exc`` is "other" (all 376 integration failures on 2026-10-10,
-# 222 of them AssertionErrors). The two routing decisions below that ask "is
-# this really an assertion?" read the exception from that prefix instead. The
-# grouping and the fingerprint still use ``terminal_exc``; changing those would
-# re-key every open group, so it is a separate change.
-_LINE_PREFIXED_EXC = re.compile(
-    r"^\(line \d+\)\s+(?:[\w.]+\.)?([A-Za-z_]\w*(?:Error|Exception))\b"
-)
-
-
-def routing_exc(target: dict) -> str:
-    """The group's terminal exception for routing: ``terminal_exc`` when it was
-    parsed, else the exception every failure's ``(line N)`` trace names."""
-    exc = target.get("terminal_exc") or ""
-    if exc not in ("", "other"):
-        return exc
-    seen = set()
-    for f in target.get("failures") or []:
-        trace = (f.get("latest_trace") or f.get("trace") or "").strip()
-        m = _LINE_PREFIXED_EXC.match(trace)
-        seen.add(m.group(1) if m else None)
-    return seen.pop() or exc if len(seen) == 1 else exc
-
-
 def dispatch_category(target: dict) -> str:
     """The category a group is really in, for selection diversity.
 
@@ -1265,7 +1246,7 @@ def dispatch_category(target: dict) -> str:
     if target.get("kind") != "model_failures":
         return "cluster"
     mode = target.get("failure_mode") or "other"
-    if mode in _CRASH_MODES and routing_exc(target) == "AssertionError":
+    if mode in _CRASH_MODES and target.get("terminal_exc") == "AssertionError":
         return "output_mismatch"
     return mode
 
@@ -2347,6 +2328,15 @@ def target_tests(target: dict) -> set[str]:
     return {f["test"] for f in target.get("failures") or [] if f.get("test")}
 
 
+# Fixing `_BARE_EXC_PAT` for the CI dataset's "(line N)" traces (2026-10-12)
+# reclassified about a fifth of the groups (other → output_mismatch), which
+# changed their labels and so their fingerprints: 68 of 110 on the 2026-10-11
+# window. A closed PR opened before that date can only be found by the tests it
+# lists, so it matches by test overlap, which keeps its rejection and the
+# maintainer feedback attached to it (big_bird's #48437, "do not record <unk>").
+FINGERPRINT_REKEY_DATE = "2026-10-12"
+
+
 def _pr_matches(pr: dict, markers: list[str], prefixes: list[str]) -> bool:
     body = pr.get("body") or ""
     head_ref = (pr.get("head") or {}).get("ref") or ""
@@ -2371,8 +2361,9 @@ def classify_prior_attempts(
     tests…", a test flaking on one runner). Each move minted a new fingerprint and
     so a second PR for the same test: on 2026-10-11 six failure groups had two
     open serge PRs each. So an open ITF PR whose body lists any of the group's tests is
-    its existing PR too, and the group gets a follow-up on it. Closed PRs still
-    match by fingerprint only, so the rejection count keeps its meaning.
+    its existing PR too, and the group gets a follow-up on it. Closed PRs match
+    by fingerprint only, so the rejection count keeps its meaning, except those
+    opened before :data:`FINGERPRINT_REKEY_DATE`, which also match by test.
 
     A PR serge closed at this triage's request (:data:`SERGE_CLOSE_MARKERS`) is
     ignored: retiring a stale or duplicate PR is not a maintainer rejecting a
@@ -2384,8 +2375,12 @@ def classify_prior_attempts(
     rejected: list[int] = []
     for pr in pulls:
         is_open = (pr.get("state") or "") != "closed"
+        pre_rekey = (pr.get("created_at") or "9999") < FINGERPRINT_REKEY_DATE
         if not _pr_matches(pr, markers, prefixes) and not (
-            tests and is_open and is_itf_pr(pr) and tests & pr_target_tests(pr)
+            tests
+            and (is_open or pre_rekey)
+            and is_itf_pr(pr)
+            and tests & pr_target_tests(pr)
         ):
             continue
         if not is_open and serge_closed_reason(pr):
@@ -4673,7 +4668,7 @@ def instruction_addendum(target: dict) -> str:
     if mode in _CRASH_MODES:
         # An `AssertionError` reaching us as `other` is still an assertion, so it
         # gets the mismatch guidance rather than "this is a library bug".
-        if routing_exc(target) == "AssertionError":
+        if target.get("terminal_exc") == "AssertionError":
             return _MISMATCH_GUIDANCE
         # A crash raised inside the weight-conversion step is a loading/placement
         # problem, not a library bug to hunt — see `_CONVERSION_PAT`. Checked only

tests/test_integration_failure_triage.py

@@ -156,6 +156,28 @@ def test_assertion_error(self):
     def test_none_when_no_exception(self):
         self.assertEqual(itf.terminal_exception("just a plain line"), (None, ""))
 
+    def test_ci_dataset_line_prefix(self):
+        """Every trace in the CI dataset's model_results.json reads
+        "(line N)  <Exc>: msg" (all 376 integration failures on 2026-10-10)."""
+        self.assertEqual(
+            itf.terminal_exception("(line 130)  AssertionError: 'a' != 'b'"),
+            ("AssertionError", "'a' != 'b'"),
+        )
+        self.assertEqual(
+            itf.terminal_exception(
+                "(line 88)  torch.OutOfMemoryError: CUDA out of memory"
+            ),
+            ("OutOfMemoryError", "CUDA out of memory"),
+        )
+
+    def test_line_prefixed_assertion_is_an_output_mismatch(self):
+        # 86 of the 222 AssertionErrors on 2026-10-10 classified `other`, so
+        # those groups were sent the crash guidance.
+        self.assertEqual(
+            itf.classify("(line 130)  AssertionError: 'x' != 'y'"), "output_mismatch"
+        )
+        self.assertEqual(itf.classify("(line 5)  ValueError: bad shape"), "other")
+
     def test_crash_site_is_raising_frame(self):
         self.assertEqual(
             itf.crash_site(_TRACE_UNBOUND_CRASH),
@@ -553,6 +575,24 @@ def test_test_overlap_never_creates_a_rejection(self):
         prior = itf.classify_prior_attempts([pr], [self.FP], {t})
         self.assertEqual(prior, itf.PriorAttempts())
 
+    def test_pre_rekey_closed_pr_keeps_its_rejection_by_test(self):
+        """The 2026-10-12 parser fix changed most fingerprints; a PR closed
+        before it can only be found by its tests (big_bird #48437)."""
+        t = "tests/models/x/test_modeling_x.py::XIntegrationTest::test_a"
+        old = {
+            "number": 48437,
+            "state": "closed",
+            "created_at": "2026-08-31T10:00:00Z",
+            "body": f"- `{t}` [single-gpu]",
+            "head": {"ref": "serge/fix/itf-aaaaaaaaaaaa-1"},
+        }
+        prior = itf.classify_prior_attempts([old], [self.FP], {t})
+        self.assertEqual(prior.rejected, (48437,))
+        newer = {**old, "number": 50000, "created_at": "2026-10-20T10:00:00Z"}
+        self.assertEqual(
+            itf.classify_prior_attempts([newer], [self.FP], {t}).rejected, ()
+        )
+
     def test_test_overlap_ignores_non_itf_branches(self):
         t = "tests/models/x/test_modeling_x.py::XIntegrationTest::test_a"
         pr = {"number": 5, "state": "open", "body": f"`{t}`", "head": {"ref": "fix-x"}}
@@ -2835,29 +2875,18 @@ def test_mismatch_names_the_runner_key_and_the_protected_ones(self):
             self.assertIn(protected, text)
 
     def test_line_prefixed_assertion_gets_the_mismatch_guidance(self):
-        """The CI dataset writes "(line 130)  AssertionError: ...", which
-        terminal_exception() does not parse, so real groups carry
-        terminal_exc "other" (transformers#49336). Before routing_exc they got
-        the CRASH block and no rule about which key to write."""
-        target = _target("other", exc="other")
-        target["failures"][0]["latest_trace"] = (
-            "(line 130)  AssertionError: 'My favourite condiment is' != 'olive oil'"
-        )
-        self.assertEqual(itf.routing_exc(target), "AssertionError")
+        """The CI dataset writes "(line 130)  AssertionError: ...". Before
+        2026-10-12 terminal_exception() could not parse that, so real groups
+        carried terminal_exc "other" and got the CRASH block (transformers#49336).
+        Built through cluster_failures/pick_targets, as the nightly builds it."""
+        trace = "(line 130)  AssertionError: 'My favourite condiment is' != 'olive oil'"
+        f = _failure("ministral3", "multi", "tests/models/ministral3/t.py::T::t", trace)
+        f.pop("failure_mode", None)
+        targets = itf.pick_targets(itf.cluster_failures([f], None, attribution={}))
+        target = next(t for t in targets if t.get("kind") == "model_failures")
+        self.assertEqual(target["terminal_exc"], "AssertionError")
+        self.assertEqual(target["failure_mode"], "output_mismatch")
         self.assertEqual(itf.instruction_addendum(target), itf._MISMATCH_GUIDANCE)
-        self.assertEqual(itf.dispatch_category(target), "output_mismatch")
-
-    def test_line_prefixed_crash_keeps_the_crash_guidance(self):
-        target = _target("other", exc="other")
-        target["failures"][0]["latest_trace"] = "(line 88)  ValueError: bad shape"
-        self.assertEqual(itf.routing_exc(target), "ValueError")
-        self.assertNotIn("#48198", itf.instruction_addendum(target))
-
-    def test_mixed_exceptions_do_not_route_as_an_assertion(self):
-        target = _target("other", exc="other", n=2)
-        target["failures"][0]["latest_trace"] = "(line 1)  AssertionError: x"
-        target["failures"][1]["latest_trace"] = "(line 2)  ValueError: y"
-        self.assertEqual(itf.routing_exc(target), "other")
 
     def test_the_convention_stays_out_of_a_crash_group(self):
         """A crash never got as far as comparing values, so "add a device key"

utils/prompt_ab.py

@@ -114,24 +114,24 @@
             }
         ],
     },
-    # transformers#49336, as the triage really builds it: the CI dataset's
-    # "(line N)  AssertionError" trace leaves terminal_exc "other", which before
-    # routing_exc sent this expectation group the CRASH guidance ("do not edit
-    # the test") and so no rule on which Expectations key to write.
+    # transformers#49336, as the triage builds it since 2026-10-12. Before that,
+    # the CI dataset's "(line N)  AssertionError" trace left terminal_exc
+    # "other" and the mode `other`, so this expectation group got the CRASH
+    # guidance ("do not edit the test") and no rule on which key to write.
     "ministral3-line-prefixed-assertion": {
         "kind": "model_failures",
-        "label": "2 integration tests for model `ministral3` failing with `other`",
+        "label": "2 integration tests for model `ministral3` failing with `output_mismatch`",
         "model": "ministral3",
-        "failure_mode": "other",
-        "terminal_exc": "other",
+        "failure_mode": "output_mismatch",
+        "terminal_exc": "AssertionError",
         "failures": [
             {
                 "test": (
                     "tests/models/ministral3/test_modeling_ministral3.py"
                     "::Ministral3IntegrationTest::test_model_3b_generation"
                 ),
                 "gpu": "multi-gpu",
-                "failure_mode": "other",
+                "failure_mode": "output_mismatch",
                 "latest_trace": (
                     "(line 130)  AssertionError: 'My favourite condiment is 100% "
                     "pure oliv[64 chars] can' != \"My favourite condiment is 100% "