From f29d114a87f3ef1b4b8d64df1e2d570599ed8383 Mon Sep 17 00:00:00 2001 From: akoita Date: Sun, 6 Sep 2026 03:44:47 +0200 Subject: [PATCH] fix(author): require whole numeric values for factual grounding --- docs/roadmap.md | 36 +++++++++++++++---- .../src/author-evidence-completion.test.ts | 31 ++++++++++++++++ .../src/author-evidence-completion.ts | 12 +++---- packages/application/src/author-grounding.ts | 14 +++++++- packages/application/src/complete-cv.ts | 4 +-- 5 files changed, 79 insertions(+), 18 deletions(-) diff --git a/docs/roadmap.md b/docs/roadmap.md index 7cab70f..2bf6a88 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -1,7 +1,7 @@ # Product vision and roadmap **Status:** Living document
-**Last reviewed:** 2026-09-01
+**Last reviewed:** 2026-09-06
**Current stage:** Workflow parity and release (v0.9.0) This document describes product direction, not fixed delivery dates. **Now** is @@ -384,32 +384,41 @@ approved fresh retry timed out during authoring because the user-session adapter's 120-second request limit could not use the 20-minute case budget. No revised, approved, or exported artifact was produced. The private manual baseline also targets a related but different role, which limits any future -comparison. The bounded provider-timeout, author-grounding, and corrective +comparison. + +The bounded provider-timeout, author-grounding, and corrective retry blockers #256, #258, and #260 are closed. The corrective observation under issue #262 is closed as indeterminate: it used all three author attempts, with output-token excess followed by two different single claim-text factual-invariant failures. No draft reached the critic. The per-evidence -author-grounding guide blocker #264 then closed. The authorized observation +author-grounding guide blocker #264 then closed. + +The authorized observation under #266 repeatedly failed before author output because structured Claude nonzero-exit errors were classified generically. Issue #267 delivered the bounded classification fix. Keep #75 unvalidated; another authorized comparison under #271 then reached all three author attempts. Output-token excess was followed by three claim-text factual-invariant paths; the final corrective attempt reduced them to one repeated path, but no proposal reached -the critic. Issue #272 delivered bounded exact-value citation completion: local +the critic. + +Issue #272 delivered bounded exact-value citation completion: local normalization may append only exact supporting retrieved chunks, while the unchanged validator still rejects unsupported values. The authorized post-completion observation under #275 then completed both provider roles on their first attempts and reached the human boundary. Readiness remained false with eight warnings and one unresolved chronology omission; the unsupported- claim finding was a warning rather than a factuality error. No adjudication, -revision, approval, export, or submission occurred. Issue #277 stages the +revision, approval, export, or submission occurred. + +Issue #277 stages the candidate's exact adjudication through the shared application/local boundary, and #278 preserves active provider-duration accounting across human waits and restart boundaries. The fresh observation under #286 used 307 seconds of active time, reached an accepted author draft on attempt three, and completed independent critique on attempt one. Its twelve findings were materially different from the previously confirmed nine, so no adjudication was applied. + Typed history then rejected the distinct new artifact because another run in the workspace already held version 1. Migration 26 under #287 removed that storage blocker. Under #290, the candidate then confirmed one accept, two @@ -417,12 +426,15 @@ rejects, and nine nuanced decisions; the exact package persisted before any provider call. The first revision attempt used an incorrect API-key default, and the two authenticated user-session attempts retained the 120-second per-request default and timed out. No revised artifact or new critique exists. + The fresh observation under #291 applied the explicit 20-minute request timeout to every user-session subprocess. Authoring completed on attempt three and the critic on attempt one. After the candidate confirmed two accepts, two rejects, and six nuanced decisions, all three revision calls returned before timeout but failed structured-response validation. The run exhausted after 959 active -seconds without a revised artifact or second critique. Issue #293 delivered +seconds without a revised artifact or second critique. + +Issue #293 delivered content-free failure-stage classification (`transport-parsing`, `response-schema-validation`, `artifact-schema-validation`, `factual-invariant-rejection`) and sanitized deterministic 10-finding carrier @@ -431,11 +443,21 @@ exercised that classification during a fresh same-scope author attempt: all three author calls returned before timeout but failed local factual-invariant checks, correctly classified under `factual-invariant-rejection`. The run exhausted after 313 seconds of active provider time without accepted author -output, so the critic was not called. The tenth observation under #298 also +output, so the critic was not called. + +The tenth observation under #298 also exhausted three author attempts across token-budget excess and `factual-invariant-rejection` after 412 active seconds. Keep #75 unvalidated and leave #250 blocked. +Issue #300 tightens numeric grounding in citation completion and author +validation: a number must match a whole source value, including its decimal +part, percentage marker, or magnitude suffix. For example, `2020` cannot +support a claimed metric of `20`. Deterministic regression coverage establishes +this bounded safety correction; it does not explain the live author failures +or change the indeterminate parity result. The next pilot observation remains +pending, and release preparation stays blocked on #75. + **Exit criterion:** The representative comparison records no factual-invariant violations or unsupported model-added facts, preserves required sections and chronology, meets the agreed relevance and coverage thresholds, and produces a diff --git a/packages/application/src/author-evidence-completion.test.ts b/packages/application/src/author-evidence-completion.test.ts index 6c3c282..d690fa0 100644 --- a/packages/application/src/author-evidence-completion.test.ts +++ b/packages/application/src/author-evidence-completion.test.ts @@ -44,6 +44,37 @@ function singleClaimProposal( } describe("author evidence citation completion", () => { + it.each(["2020", "120", "20.5", "1,020", "20%", "20k", "20m", "20b"])( + "does not ground a metric of 20 with the different numeric value %s", + (sourceValue) => { + const evidence = [chunk("chunk-number", `delivered ${sourceValue} projects`)]; + const uncited = singleClaimProposal("delivered 20 projects"); + expect(completeAuthorEvidenceCitations(uncited, evidence)).toBe(uncited); + + const cited = singleClaimProposal("delivered 20 projects", true, ["chunk-number"]); + expect(completeCvProposalIssues(cited, evidence)).toEqual([ + expect.objectContaining({ code: "factual_invariant_violation" }), + ]); + }, + ); + + it.each([ + ["20", "delivered 20 projects."], + ["20.5", "delivered 20.5 projects."], + ["20%", "delivered 20% of projects."], + ["20k", "delivered 20K projects."], + ["2020", "delivered projects in 2020-2024."], + ["20", "delivered ２０ projects."], + ])("retains exact numeric support for %s in %s", (value, sourceText) => { + const evidence = [chunk("chunk-number", sourceText)]; + const completed = completeAuthorEvidenceCitations( + singleClaimProposal(`delivered ${value} projects`), + evidence, + ); + expect(completed.sections[0]?.blocks[0]?.claims[0]?.evidenceChunkIds).toEqual(["chunk-number"]); + expect(completeCvProposalIssues(completed, evidence)).toEqual([]); + }); + it("matches protected values with exact NFKC and en-US lowercase comparison", () => { const text = "Staff Engineer"; const proposal = singleClaimProposal(text); diff --git a/packages/application/src/author-evidence-completion.ts b/packages/application/src/author-evidence-completion.ts index 71f8495..a3224c2 100644 --- a/packages/application/src/author-evidence-completion.ts +++ b/packages/application/src/author-evidence-completion.ts @@ -1,11 +1,7 @@ import type { ScoredEvidenceChunk } from "@draft-loop/domain"; import type { AuthorArtifactProposal } from "@draft-loop/schemas"; -import { extractProtectedValues } from "./author-grounding.js"; - -function normalizedForComparison(value: string): string { - return value.normalize("NFKC").toLocaleLowerCase("en-US"); -} +import { extractProtectedValues, supportsProtectedValue } from "./author-grounding.js"; function completedEvidenceChunkIds( claim: AuthorArtifactProposal["sections"][number]["blocks"][number]["claims"][number], @@ -25,10 +21,10 @@ function completedEvidenceChunkIds( } for (const chunk of retrievedEvidence) { - const supportsProtectedValue = protectedValues.some((value) => - normalizedForComparison(chunk.text).includes(normalizedForComparison(value)), + const supportsAnyValue = protectedValues.some((value) => + supportsProtectedValue(chunk.text, value), ); - if (!supportsProtectedValue || seen.has(chunk.id)) continue; + if (!supportsAnyValue || seen.has(chunk.id)) continue; seen.add(chunk.id); completedIds.push(chunk.id); } diff --git a/packages/application/src/author-grounding.ts b/packages/application/src/author-grounding.ts index 915d27e..3a91793 100644 --- a/packages/application/src/author-grounding.ts +++ b/packages/application/src/author-grounding.ts @@ -1,9 +1,11 @@ import type { ScoredEvidenceChunk } from "@draft-loop/domain"; +const protectedNumberPattern = /(? match[0] === value); + } + return source.includes(value); +} + /** Extract exact protected values in first-occurrence order without duplicates. */ export function extractProtectedValues(value: string): readonly string[] { const matches: ProtectedValueMatch[] = protectedValuePatterns.flatMap((pattern, patternIndex) => diff --git a/packages/application/src/complete-cv.ts b/packages/application/src/complete-cv.ts index 0606233..b4324d7 100644 --- a/packages/application/src/complete-cv.ts +++ b/packages/application/src/complete-cv.ts @@ -1,7 +1,7 @@ import type { ScoredEvidenceChunk } from "@draft-loop/domain"; import type { AuthorArtifactProposal } from "@draft-loop/schemas"; -import { extractProtectedValues } from "./author-grounding.js"; +import { extractProtectedValues, supportsProtectedValue } from "./author-grounding.js"; export const factualInvariantIssueCodes = [ "missing_evidence", @@ -68,7 +68,7 @@ export function completeCvProposalIssues( }); } for (const value of extractProtectedValues(claim.text)) { - if (!evidence.includes(normalized(value))) { + if (!supportsProtectedValue(evidence, value)) { issues.push({ code: "factual_invariant_violation", path: [...path, "text"],