From 45c52b9f1226cb31448639fbfcfb69539703504e Mon Sep 17 00:00:00 2001 From: Sam Paniagua Date: Wed, 12 Aug 2026 12:33:02 +0200 Subject: [PATCH 1/4] feat: complete Knolo v4 Phase 5 TrustBench --- .github/workflows/ci.yml | 1 + .gitignore | 1 + README.md | 47 +++-- conformance/README.md | 36 ++++ conformance/expected/retrieval-v4.0.json | 125 ++++++++++++++ .../packs/corrupted/section-digest.knolo | Bin 0 -> 4968 bytes conformance/packs/corrupted/truncated.knolo | Bin 0 -> 4961 bytes conformance/packs/minimal-v1.knolo | Bin 0 -> 89 bytes conformance/packs/standard-v3.knolo | Bin 0 -> 851 bytes conformance/packs/verified-v4.knolo | Bin 0 -> 4968 bytes conformance/queries/retrieval.jsonl | 5 + examples/icp-knowledge-canister/README.md | 4 + examples/langchain-basic/README.md | 5 +- examples/llamaindex-basic/README.md | 5 +- examples/memory-overlay/README.md | 4 + examples/nextjs-rag-chat/README.md | 5 +- examples/node-cli-rag/README.md | 4 +- package-lock.json | 8 +- package.json | 2 + packages/cli/README.md | 19 ++- packages/cli/bin/knolo.mjs | 102 ++++++++++- packages/cli/package.json | 4 +- .../icp-knowledge-canister/README.md | 4 + packages/cli/test/cli.test.mjs | 34 ++++ packages/core-python/README.md | 9 +- packages/core-python/RELEASE.md | 5 + packages/core-python/tests/fixtures/README.md | 4 + packages/core-rust/README.md | 4 + packages/core/CHANGELOG.md | 41 +++++ packages/core/README.md | 7 + packages/core/package.json | 2 +- packages/core/scripts/test.mjs | 69 +++++++- packages/core/src/analyzer.ts | 25 +++ packages/core/src/builder.ts | 30 ++-- packages/core/src/index.ts | 14 +- packages/core/src/pack.runtime.ts | 43 ++++- packages/core/src/pack.ts | 3 +- packages/core/src/pack.v4.ts | 144 ++++++++++++++++ packages/core/src/quality/diversify.ts | 2 +- packages/core/src/query.ts | 160 +++++++++++++----- packages/core/src/rank.ts | 6 +- packages/core/src/receipt.ts | 79 +++++++++ packages/core/src/retrieval_plan.ts | 21 +++ packages/core/src/semantic/types.ts | 10 ++ packages/core/src/utils/sha256.ts | 50 ++++++ packages/create-knolo-app/README.md | 7 +- packages/create-knolo-app/template/README.md | 5 +- .../create-knolo-app/template/package.json | 4 +- packages/icp-canister/CHANGELOG.md | 7 + packages/semantic-ollama/package.json | 2 +- tests/icp-local/README.md | 4 + trustbench/evaluator.mjs | 37 ++++ trustbench/generate-fixtures.mjs | 34 ++++ trustbench/package.json | 10 ++ trustbench/run.mjs | 13 ++ trustbench/test.mjs | 17 ++ 56 files changed, 1178 insertions(+), 105 deletions(-) create mode 100644 conformance/README.md create mode 100644 conformance/expected/retrieval-v4.0.json create mode 100644 conformance/packs/corrupted/section-digest.knolo create mode 100644 conformance/packs/corrupted/truncated.knolo create mode 100644 conformance/packs/minimal-v1.knolo create mode 100644 conformance/packs/standard-v3.knolo create mode 100644 conformance/packs/verified-v4.knolo create mode 100644 conformance/queries/retrieval.jsonl create mode 100644 packages/core/src/analyzer.ts create mode 100644 packages/core/src/pack.v4.ts create mode 100644 packages/core/src/receipt.ts create mode 100644 packages/core/src/retrieval_plan.ts create mode 100644 packages/core/src/utils/sha256.ts create mode 100644 trustbench/evaluator.mjs create mode 100644 trustbench/generate-fixtures.mjs create mode 100644 trustbench/package.json create mode 100644 trustbench/run.mjs create mode 100644 trustbench/test.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bac5ab8..60d6a34 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,3 +16,4 @@ jobs: - run: npm ci - run: npm run build - run: npm test + - run: npm run trustbench:test diff --git a/.gitignore b/.gitignore index 288e3dc..1f02b26 100644 --- a/.gitignore +++ b/.gitignore @@ -66,6 +66,7 @@ Desktop.ini # Local rollout notes (not part of the public repository) LIVE_KBS_MVP.md +/upcoming feature.md # ICP local harness generated seed data / runtime (also covered by tests/icp-local/.gitignore) tests/icp-local/data/ diff --git a/README.md b/README.md index 44aa57f..4188ce6 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,9 @@ # 🧠 Knolo +**Current contract: Knolo v4 / Phase 5 TrustBench.** TypeScript `@knolo/core` +builds v4 packs by default. v1–v3 packs remain readable; Python, Rust, and ICP +are legacy compatibility profiles until their v4 conformance runners land. + Knolo is a **local-first knowledge base engine** built around deterministic retrieval and portable `.knolo` packs. It provides: @@ -20,35 +24,38 @@ Knolo prioritizes: * Portable binary knowledge packs * Optional ICP deployment for on-chain knowledge retrieval * Strict runtime contracts (optional advanced features) +* Verifiable receipts, evidence spans, and deterministic retrieval-plan hashes > ⚠️ `knolo-core` (unscoped) on npm is deprecated. Use `@knolo/core`. --- -# πŸ“Š Retrieval Benchmark (March 2026) +# πŸ“Š TrustBench Reference (v4 / Phase 5) -Knolo was evaluated using a deterministic lexical-first + optional rerank configuration. +Knolo is evaluated using the checked-in deterministic lexical-first reference +profile and the shared `conformance/` fixtures. -**Run:** 2026-03-01 -**TopK:** 5 +**Contract:** `retrieval-v4.0` +**Runner:** `npm run trustbench:test` ### Aggregate Metrics | Metric | Score | | ----------- | --------- | -| Precision@5 | **0.490** | -| Recall@5 | **1.000** | -| MRR@5 | **0.867** | -| nDCG@5 | **0.900** | +| Metric | Definition | +| --- | --- | +| Recall@K | Relevant fixture sources retrieved | +| MRR@K | Reciprocal rank of the first relevant source | +| nDCG@K | Discounted ranking quality | +| Abstention precision | Correct empty-scope decisions | ### Interpretation -* βœ… **Recall@5 = 1.0** β†’ All relevant documents were retrieved in every test query. -* βœ… **High MRR (0.867)** β†’ Relevant documents appear near the top. -* βœ… **Strong nDCG (0.900)** β†’ Ranking quality is consistently high. -* πŸ” Precision reflects lexical grounding before rerank β€” by design, Knolo prioritizes deterministic recall over aggressive pruning. +* Results are generated from committed fixtures, not a time-stamped external benchmark. +* IDs, plan hashes, decisions, receipts, and corruption rejection are canonical. +* Scores are rounded to six decimals for cross-runtime comparison. -This benchmark demonstrates: +This contract demonstrates: * Deterministic lexical retrieval is highly reliable. * Hybrid reranking improves ranking quality without sacrificing grounding. @@ -212,12 +219,24 @@ cargo test --- -# 🐍 Python Runtime Support (Phase 2) +# 🐍 Python Runtime Support (Legacy compatibility profile) Knolo also ships a pure-Python runtime in `packages/core-python` for mounting existing `.knolo` packs and running deterministic lexical queries locally. It stays local-first, requires no vector database, and does not use embeddings on the default query path. +The Python package currently reads v1–v3 packs and is not yet a v4 TrustBench-equivalent runtime. + +# πŸ§ͺ TrustBench / Conformance + +```bash +npm run trustbench:generate +npm run trustbench:test +``` + +The suite checks deterministic IDs, scores, plan hashes, receipts, Recall@K, +MRR, nDCG, abstention precision, and fail-closed corruption handling. + Install locally: ```bash diff --git a/conformance/README.md b/conformance/README.md new file mode 100644 index 0000000..cfeea78 --- /dev/null +++ b/conformance/README.md @@ -0,0 +1,36 @@ +# Knolo conformance fixtures + +This directory defines the shared Phase 5 contract. Fixtures are generated by +`node trustbench/generate-fixtures.mjs` so binary artifacts are reproducible +and do not depend on a developer's machine. + +```text +conformance/ + packs/{minimal-v1,standard-v3,verified-v4}.knolo + packs/corrupted/{truncated,section-digest}.knolo + queries/retrieval.jsonl + expected/retrieval-v4.0.json +``` + +Each query declares an ID, query text, options, and optional relevance labels. +The canonical evaluator records hit IDs, scores rounded to six decimals, +decision/rejection status, plan hashes, receipt verification, Recall@K, MRR, +nDCG, and abstention precision. Exact scores are compared with a declared +tolerance; IDs, decisions, plan hashes, and rejection behavior are exact. + +Supported runtime profiles are recorded in the output. A profile must not be +marked equivalent until it consumes this same fixture set and canonicalizes +the same result shape. + +## Commands + +From the repository root: + +```bash +npm run trustbench:generate # rebuild packs and expected output +npm run trustbench:test # fail on result drift or corruption acceptance +``` + +The TypeScript profile is the reference output. Runtime adapters should add a +profile runner under `trustbench/profiles/` only after they can consume the v4 +fixture and emit equivalent IDs, decisions, plan hashes, and receipt state. diff --git a/conformance/expected/retrieval-v4.0.json b/conformance/expected/retrieval-v4.0.json new file mode 100644 index 0000000..a8bcb90 --- /dev/null +++ b/conformance/expected/retrieval-v4.0.json @@ -0,0 +1,125 @@ +{ + "contract": "retrieval-v4.0", + "packVersion": 4, + "packFormat": "v4", + "aggregate": { + "recallAtK": 0.7, + "mrrAtK": 0.8, + "ndcgAtK": 0.922629, + "meanHitCount": 1, + "answerCount": 4, + "abstentionCount": 1, + "abstentionPrecision": 1, + "receiptsVerified": true + }, + "rows": [ + { + "id": "lexical-alpha", + "query": "alpha beta", + "hits": [ + { + "id": "intro.md", + "blockId": 0, + "score": 0.956856 + }, + { + "id": "runtime.md", + "blockId": 1, + "score": 0.797493 + } + ], + "decision": "answer", + "planHash": "sha256-49e6fa300d07005e2bd75c0856d1c182fb9fd3cf36ca5b29308d0e1012ee18eb", + "receiptVerified": true, + "metrics": { + "recallAtK": 0.5, + "reciprocalRank": 1, + "ndcgAtK": 0.613147, + "hitCount": 2, + "relevantCount": 1 + } + }, + { + "id": "namespace-alpha", + "query": "alpha", + "hits": [ + { + "id": "intro.md", + "blockId": 0, + "score": 1.069768 + } + ], + "decision": "answer", + "planHash": "sha256-101807cf8e82cd778cf944f7cbb144943645d13468a196a3c0a60075ec460c24", + "receiptVerified": true, + "metrics": { + "recallAtK": 0.5, + "reciprocalRank": 1, + "ndcgAtK": 1, + "hitCount": 1, + "relevantCount": 1 + } + }, + { + "id": "source-runtime", + "query": "runtime", + "hits": [ + { + "id": "runtime.md", + "blockId": 1, + "score": 2.36473 + } + ], + "decision": "answer", + "planHash": "sha256-2ec416027feeeb28bcdca0d3a04ffc4c6525edce1b9dc6db70a95ce67890b016", + "receiptVerified": true, + "metrics": { + "recallAtK": 1, + "reciprocalRank": 1, + "ndcgAtK": 1, + "hitCount": 1, + "relevantCount": 1 + } + }, + { + "id": "required-phrase", + "query": "alpha", + "hits": [ + { + "id": "intro.md", + "blockId": 0, + "score": 1.441785 + } + ], + "decision": "answer", + "planHash": "sha256-32734c7016b6298c1967bf54fd04f053f37cf01b5df1dcd3efa47e48d26dc31b", + "receiptVerified": true, + "metrics": { + "recallAtK": 0.5, + "reciprocalRank": 1, + "ndcgAtK": 1, + "hitCount": 1, + "relevantCount": 1 + } + }, + { + "id": "empty-scope", + "query": "alpha", + "hits": [], + "decision": "abstain", + "planHash": "sha256-fb84e31df21e151c53999de7ac6a8323b3f47630dc589ff94ccc02082f8883f8", + "receiptVerified": true, + "metrics": { + "recallAtK": 1, + "reciprocalRank": 0, + "ndcgAtK": 1, + "hitCount": 0, + "relevantCount": 0 + } + } + ], + "runtime": { + "name": "typescript", + "version": "v20.8.0" + } +} diff --git a/conformance/packs/corrupted/section-digest.knolo b/conformance/packs/corrupted/section-digest.knolo new file mode 100644 index 0000000000000000000000000000000000000000..e88bc2e9c68d7fa82171e9a0cc34e3a7dd352464 GIT binary patch literal 4968 zcmeHLOKTiQ5LO-p@-M`kS9ZUqA38aPkeHYRY~zd2p}MPDJMr$!W@aQ?5xV5&f1T=m z$SbWBBIFP-8d$bps;&O2>Z@<`%dangdu#UgzTMl~`}qCd-go@{j3*z<0zbnPRBaE8 z1c?U0Djkw0jY)zr5xiH7R%=%Q+{ zI;@&@?7Js(tR@vUCp3(_w!V^t-~)lTvKG?GWW9^RDVLS>L>OZ*$yi&1LkLj-qKvsl zvVxDqEZO9ouz@${3u3DyeHfoEX3(^}(ZZaD1*EcpT^gRiY%vAiSU^8e|90bY^{(w^ z-9ZoU56+4~Xkm8#=k9^c`crBTMoQDRNigF-dA7bwb|9h$VXTl=DB-0K5gb`CI!hgF zKuH-CPowvSWbLvELU`dKWJCPZX?oheYuc0Q@S+;V9!5GjUwMBzr*`BMS3p;?y1bmM z5$;dhv6<8D7?V^SQ01dU$|7g7&ekM@1!pPPT4P;%sh!MdYD(xR1RW5)6TUV+@vr`S zj3bR-CdA!1rT%t+ACve&Nql_CL{W;BGkwnKgRo4YAx&ux)uEhJ)TR!XWl{Y;Fw0D; zp8hy(dSZ4N`%~h=fd#~7){M=Pc#bDkm-9f(G>RE=Qq8D68K*o8dY6Q1@ET%LQM#nG z&z zX36`(@BEhgQrcbr69nTCtxHiy%M4}e=#|mj5J=j{tb`GT^^j4yL=kmPV1uU+F=2vj z|Ca=MxA*tY9_Hsy=m&92_BbnDh&~2ww2|2d8>NWRWKD`BNp?F)tc_Ak`_5w>Bu9)J zK&|+y*WsT-+vL(9?}Go{7tp@=yh$G^yx$GLLv_Wq0C4l2-y;ASX2Wj%R%vBstB#Ei zTXpQ*?llybW8kKmR7!!Hu>~sygY8L1!L@(_k`KwqsGUIPxGKB@^4fW=IYQOt@pU1y zY51iZFNMrXC~TH*>HrSfRd;4rZ%Um9%Y#c!iV;Q@Ru@Y>Qyp?-GnJuH|Lw=T_Yw;dJ=<*X-Ut;Qohp1XbyMYf$W*&Q8K zYb7O*Cr8z)l17SmWk(YZrREW&5&@t^+VB&`!9-u-M?mG z%4&GklD3X;3b4|AaYOXpDMYy}IX)cL1@NFqKIqxif_=+2WTBp>rPr(FlKwpVak<4o zVWC)lh<|V+4!KLx>+^Z+W*oPp3TRqoYBwhG|S$yxzUbnbq!E~W`7yN{j1<{ z7Hyhh#Jug;Xt@P*;?H!Y!klQ}H1>H3@y8~YFfe$aS+9d%o4vhn_xAQae!sW(9shm7laFPApJ58Bwg*On zM1x?J4oQ>7B*7SoAVMvZOh68m2OE8jC=3N9G-wx*bnR*sD`TJRq4puyA^QZbHUU)# z!J~1}TEQlhAWJ5YQC1m?E)$@Z386ORRl12FuV9`_rPZTIkg8PrD@wFnDM{7Sl=Z(5YdA$R!A$9@Y06}jw~3Rr4BZr zqzsCu(R)L(cG(0Wyl@e+A%1n5p0*#F_M|$ztcJ0NkxnjF-k;8?9r?sH(3PyNt|n`Q z`_p!8=5#m4Bozl#`6!XH$eFCOHOXMXSqiq+Sl3=^Co`Iw5;_V&2So3LuZ>Uq)_;$2 zr18syxErU`-wp6n5kG{2yh%$JKbQYbGpBC+uf?wW zStfH)BZ@Dj}-Ss~~Fdosm6m_)BP^OMv8O;rWq>ao<7*SXc8I?;EQRf6UcnT2{CfN3W zNuc+8|M2W#e*T1h5O-vcv(km=W6(w$nSHQPiWp7Sq)3uvx0A%$D8;nzJk~*S#JB;} zim!ST{yDTwE)DWN_#b=$?TgQw^pV2*-2gmRSKJ5yH{baK0+3-g?AC9UR%W*9*!Zwj z$Ik6uLvc9*r8p+ZmJH(NVQl zQu26mRINI`D#H_)GhU$Ii${EBG&W<4+C@_fy|vS0p|XEQJ3a(1)6 zJ>Om{GP%sl+0C;SpL~xZ3fc9g&--hAg3Z7(@#5VbTe+bb60Wy?D4KEqRZ+0}H!Mt9 z4X;|#))7ttR+_JFiQapKD3>M2hr_x69u&z3J-b@4Z`p<{)YG)|cC}p6pJzWVw>T&) z6w8bFg&T3mU6S5j%wsp>xE&oAr8`d7!x7?)erBgx_Lj|!c4Vt-fC@DG%Mk8g2amIO z8>l;v+yv}pvGQi6N0(^tnEZs?_Sr4yZm8KE@QM$7c!?cn{e9Rm&;94vap*ona=oEU zJxd(==LEYxwuRJ|DBZznnXMb3{;wpv#i&o8hEJb{U*6aERu$fE`cI#RkNh;eQ=F96 gwpJ%)$bFE`OaGifq}jW!^xsQvkliUa(=urO2HLvjmjD0& literal 0 HcmV?d00001 diff --git a/conformance/packs/minimal-v1.knolo b/conformance/packs/minimal-v1.knolo new file mode 100644 index 0000000000000000000000000000000000000000..6bebfa28a6a823d025aead27544a9f9fd268300d GIT binary patch literal 89 zcmXqIU|^_LDoZUY&dkqKvNF_BDlSPZDOR$oR!YfF2GT&0q@4WZYzVU?wI~P-cg_gROQN)9vk9QLPPD~`&gn!zW3l$Bj_FIRi zTBpsYY=9%uOLD%Ps3&g2g`aksSGo&AwpCo#WU3b?kP^k=)s~TreHL(u=5n8G7|PF4Tmj(nk(| z6)vlLZ9Bv7#&X`v11s@2ZOI{xE%hfXh{K}xW?~P`Mbyz5^V#ZyL<0 Vc{C#3z)NMGS#SZ0*KQcc@i+0%p+UQdq-$5BSQ-0d548`u4%sJgwF#&~ z2p)}#)(SS61X(hHjIzpDbeRCPObE3huM*~KdzaN2^+VIO)uEYGLwDLI8mhyi%c{ZZ zuxi?|@1D)EnpD`F&@l4a`brXl4+P%IT1Y38^)3phTvpN(VT{2fV{Hu%Aw&U)GUgh| z3O*9EWRr8k2HssPh^>xvK0aT}plNxdg*goiNM!@NG(3aZVhX&mfPSF<{l?|$L)*=| zgC71oI4cIBh1tbly9YMwAaDNWlZ!Hoap#riJUfruW2u|is*gqJ=>1q3+X-}%d%W4>V80q9<<^AcL+L2FO16|4L>T0q^ zxIb;jW=?lwOj2<`m5&lBi=4?iTayeHoTXrEjdks%b~2->DWRhfbU^e@_}ciyzxwYn zjx>Im5O?F0`nv&sO5#T)@$nTCMJZO!^f{*w!ZL-1G^IIIhjLO;n>t*UMfLl@EHkNk z`s1|eiP>fBPl*c$77&|RGd4@&Ii6Hq&I2*iC}zk>HKX=qoboK_T@tFnYlulj>5|fp z>mnPWLPDvNBW5Fm8DmYtDpH1N(rNTISd2=l98Fc~aD9Opoi}Oe;^*?;Y39_8|Fzh4 zKg*;rIpM$?V?dgqf}|i_3?Va#*I9bSt)m~@k^8>N``oyR&zju-zZ-zZ>WUiy;O0AjKman#hTZzD(#p(M9UC9E z>e#v6YbY+qz)dx&lma(n3swpS+mnofYXJo$ACi$#JAuw|Rd@&FwewnYgsQ9Kn?h#O z@Jlyd3YnEq*eu`H0UWgJ?#!;m0lT%ND_Q})Yuc{cMhE@wCE z+w<+UB9qI!oZUQY@yYinqL5u*`nn%KhoTwxUlj$rf5XC* z)$pn%Z5`nhV5Rx$mgv1#h;mtSd^oHN;6agm(6g%r`<89ULOo4OZ&%AD{dxA|a*Kn) zLb3c1|KLU(a+jpH7xUQ7IBrMBMd^;y^>BnZqo3Jnmc3Cq+HJ0?FNw|#aCx*KYC2fX3~A6{a|S$`jP%ya)ab{x9TkX&yl zQ_m8I{yD*}k8L5fB}#X2T4w78sQ)X;ZZYc9tKrkD;g|RJy;X&`oBq?Q;UixS?-VDc mwXM}j8FC+_^U^ literal 0 HcmV?d00001 diff --git a/conformance/queries/retrieval.jsonl b/conformance/queries/retrieval.jsonl new file mode 100644 index 0000000..3f7db98 --- /dev/null +++ b/conformance/queries/retrieval.jsonl @@ -0,0 +1,5 @@ +{"id":"lexical-alpha","query":"alpha beta","options":{"topK":3},"relevant":["intro.md","other.md"]} +{"id":"namespace-alpha","query":"alpha","options":{"topK":5,"namespace":"docs.alpha"},"relevant":["intro.md","other.md"]} +{"id":"source-runtime","query":"runtime","options":{"topK":3,"source":"runtime.md"},"relevant":["runtime.md"]} +{"id":"required-phrase","query":"alpha","options":{"topK":3,"requirePhrases":["alpha beta"]},"relevant":["intro.md","other.md"]} +{"id":"empty-scope","query":"alpha","options":{"topK":3,"namespace":"does-not-exist"},"relevant":[]} diff --git a/examples/icp-knowledge-canister/README.md b/examples/icp-knowledge-canister/README.md index 5c3f472..a1c58bd 100644 --- a/examples/icp-knowledge-canister/README.md +++ b/examples/icp-knowledge-canister/README.md @@ -1,5 +1,9 @@ # icp-knowledge-canister +This example uses the legacy ICP v3 capability profile. TypeScript builds v4 +by default, so the example’s ICP pack builder intentionally requests v3 until +the canister supports the v4 container and TrustBench contract. + Minimal local `dfx` example for deploying the Knolo ICP canister without any middleware server. The example: diff --git a/examples/langchain-basic/README.md b/examples/langchain-basic/README.md index 9209bcd..3a0b793 100644 --- a/examples/langchain-basic/README.md +++ b/examples/langchain-basic/README.md @@ -1,5 +1,9 @@ # langchain-basic +This adapter consumes the Knolo v4 TypeScript runtime and remains a thin +retriever wrapper; receipts and retrieval plans are available from +`@knolo/core`. + Minimal LangChain-style retrieval with `@knolo/langchain` (no `langchain` dependency required). ```bash @@ -12,4 +16,3 @@ npm run start ## License Apache-2.0 - diff --git a/examples/llamaindex-basic/README.md b/examples/llamaindex-basic/README.md index 2049ee7..2405a1c 100644 --- a/examples/llamaindex-basic/README.md +++ b/examples/llamaindex-basic/README.md @@ -1,5 +1,9 @@ # llamaindex-basic +This adapter consumes the Knolo v4 TypeScript runtime and remains a thin +retriever wrapper; receipts and retrieval plans are available from +`@knolo/core`. + Minimal LlamaIndex-style retrieval with `@knolo/llamaindex` (no `llamaindex` dependency required). ```bash @@ -12,4 +16,3 @@ npm run start ## License Apache-2.0 - diff --git a/examples/memory-overlay/README.md b/examples/memory-overlay/README.md index 0f61e1b..c4e8161 100644 --- a/examples/memory-overlay/README.md +++ b/examples/memory-overlay/README.md @@ -1,5 +1,9 @@ # Knolo Cortex memory overlay +The overlay now serializes into v4 packs by default. Verify generated packs +with `knolo verify` and use `queryWithReceipt()` when evidence/replay metadata +is required. + ```ts import { buildPack, diff --git a/examples/nextjs-rag-chat/README.md b/examples/nextjs-rag-chat/README.md index c0391cf..eb53123 100644 --- a/examples/nextjs-rag-chat/README.md +++ b/examples/nextjs-rag-chat/README.md @@ -1,5 +1,9 @@ # nextjs-rag-chat +The scaffold targets Knolo v4 packs and the TypeScript TrustBench reference +runtime. Use the CLI’s `--receipt` option when the application needs verifiable +retrieval evidence. + Use the published-style scaffold package from this monorepo: ```bash @@ -20,4 +24,3 @@ npm run dev ## License Apache-2.0 - diff --git a/examples/node-cli-rag/README.md b/examples/node-cli-rag/README.md index b901bdb..dbf068f 100644 --- a/examples/node-cli-rag/README.md +++ b/examples/node-cli-rag/README.md @@ -1,5 +1,8 @@ # node-cli-rag +This example targets the v4 CLI workflow: inspect, verify, migrate, and receipt +output are available alongside normal deterministic queries. + Minimal CLI retrieval flow using workspace `@knolo/cli`. ```bash @@ -12,4 +15,3 @@ npm run build-pack && npm run query ## License Apache-2.0 - diff --git a/package-lock.json b/package-lock.json index 356ee7a..729c0b4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -66,9 +66,9 @@ }, "packages/cli": { "name": "@knolo/cli", - "version": "3.2.2", + "version": "4.0.0", "dependencies": { - "@knolo/core": "^3.2.0" + "@knolo/core": "^4.0.0" }, "bin": { "knolo": "bin/knolo.mjs" @@ -76,7 +76,7 @@ }, "packages/core": { "name": "@knolo/core", - "version": "3.5.0", + "version": "4.0.0", "license": "Apache-2.0", "devDependencies": { "@types/node": "^20.11.0", @@ -110,7 +110,7 @@ "name": "@knolo/semantic-ollama", "version": "0.1.1", "dependencies": { - "@knolo/core": "^3.2.1" + "@knolo/core": "^4.0.0" }, "devDependencies": { "typescript": "^5.5.0" diff --git a/package.json b/package.json index 501b379..d830f18 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,8 @@ "test:rust": "cargo test --manifest-path packages/core-rust/Cargo.toml", "test:icp": "cargo test --manifest-path packages/core-rust/Cargo.toml && cargo test --manifest-path packages/icp-canister/Cargo.toml && bash scripts/check-icp-template-sync.sh", "test:icp:e2e": "bash scripts/e2e-icp-local.sh", + "trustbench:generate": "npm run build --workspace @knolo/core && node trustbench/generate-fixtures.mjs && node trustbench/run.mjs", + "trustbench:test": "npm run build --workspace @knolo/core && node trustbench/test.mjs", "icp:local": "bash tests/icp-local/run-local.sh" } } diff --git a/packages/cli/README.md b/packages/cli/README.md index 103dee3..36de526 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -1,5 +1,8 @@ # @knolo/cli +The CLI targets the Knolo v4 pack contract. Normal builds emit v4; the ICP +legacy build path explicitly emits v3 until the canister gains a v4 profile. + The official CLI for building `.knolo` knowledge packs. It indexes structured content and produces a deterministic, local-first knowledge bundle for use with `@knolo/core`. @@ -36,6 +39,17 @@ Indexes your configured content and outputs: dist/knowledge.knolo ``` +Artifact workflows: + +```bash +knolo inspect dist/knowledge.knolo +knolo verify dist/knowledge.knolo +knolo migrate old.knolo --to 4 --out new.knolo +knolo query "billing policy" --receipt receipt.json --json +knolo explain receipt.json --pack dist/knowledge.knolo +knolo diff old.knolo new.knolo +``` + ### ICP Canister Workflow (LIVE) Deploy a Knolo knowledge canister on a local ICP replica, upload a `.knolo` pack, and query it with no middleware. @@ -109,12 +123,13 @@ All builds are deterministic. ## 🧠 Agent Features -Phase 2 includes: +Current v4 features include: * Routing profile extraction * Tool policy validation * Mount-time registry validation * Deterministic selection logic +* Pack manifests, analyzer identities, retrieval plans, receipts, and evidence spans --- @@ -153,4 +168,4 @@ This section is deterministic, offline-safe, and additive; runtimes that ignore ## πŸ“„ License -MIT +Apache-2.0 diff --git a/packages/cli/bin/knolo.mjs b/packages/cli/bin/knolo.mjs index cea78e5..63b2f0a 100755 --- a/packages/cli/bin/knolo.mjs +++ b/packages/cli/bin/knolo.mjs @@ -33,7 +33,7 @@ const DEFAULT_CONFIG = { }; const SUPPORTED_EXTENSIONS = new Set(['.md', '.txt', '.json']); const SKIP_DIRS = new Set(['node_modules', 'dist', '.git', '.dfx', '.cargo-target']); -const SUBCOMMANDS = new Set(['init', 'add', 'build', 'query', 'dev', 'semantic:index', 'semantic:inspect', 'semantic:validate']); +const SUBCOMMANDS = new Set(['init', 'add', 'build', 'query', 'inspect', 'migrate', 'verify', 'explain', 'diff', 'dev', 'semantic:index', 'semantic:inspect', 'semantic:validate']); const ICP_SUBCOMMANDS = new Set(['init', 'build-pack', 'upload', 'query', 'health', 'info', 'clear']); const ICP_TEMPLATE_CANDIDATES = [ path.resolve(__dirname, '../templates/icp-knowledge-canister'), @@ -82,6 +82,11 @@ Commands: add Add or update a source entry in config build Build a .knolo pack from configured sources query Query a built pack and print top hits + inspect Inspect pack format, sections, and manifest + migrate Migrate a legacy pack to v4 + verify Verify pack structure and cryptographic digests + explain Verify and explain a query receipt + diff Compare pack identities and source/chunk counts dev Watch config/sources and rebuild on change icp Scaffold and operate an ICP-native knowledge canister @@ -97,7 +102,12 @@ function printCommandHelp(command) { init: 'Usage: knolo init', add: 'Usage: knolo add ', build: 'Usage: knolo build', - query: 'Usage: knolo query [--pack ] [--k ] [--json]', + query: 'Usage: knolo query [--pack ] [--k ] [--receipt ] [--json]', + inspect: 'Usage: knolo inspect ', + migrate: 'Usage: knolo migrate --out [--to 4]', + verify: 'Usage: knolo verify ', + explain: 'Usage: knolo explain --pack ', + diff: 'Usage: knolo diff ', dev: 'Usage: knolo dev', icp: 'Usage: knolo icp [options]', 'semantic:index': 'Usage: knolo semantic:index --pack [--out ] [--model ] [--endpoint ]', @@ -531,7 +541,7 @@ async function cmdIcpBuildPack(core, args) { const rootDir = statSync(resolvedInput).isDirectory() ? resolvedInput : path.dirname(resolvedInput); const docs = buildDocsFromFiles(files, rootDir); - const bytes = await core.buildPack(docs); + const bytes = await core.buildPack(docs, { format: 3 }); const outPath = path.resolve(process.cwd(), flags.out); mkdirSync(path.dirname(outPath), { recursive: true }); @@ -636,6 +646,7 @@ function parseQueryArgs(args) { if (arg === '--json') opts.json = true; else if (arg === '--pack') opts.pack = args[++i]; else if (arg === '--k') opts.k = args[++i]; + else if (arg === '--receipt') opts.receipt = args[++i]; else if (arg.startsWith('--')) throw createError(`Unknown flag for query: ${arg}`); else positional.push(arg); } @@ -657,7 +668,12 @@ async function cmdQuery(core, args) { const packBuffer = readFileSync(packPath); const bytes = Uint8Array.from(packBuffer); const kb = await mountPackFromBytes(core, bytes); - const hits = core.query(kb, question, { topK }).map((hit) => ({ + const receiptResult = opts.receipt && core.queryWithReceipt ? core.queryWithReceipt(kb, question, { topK }) : null; + if (receiptResult && opts.receipt) { + writeFileSync(path.resolve(process.cwd(), opts.receipt), `${JSON.stringify(receiptResult.receipt, null, 2)}\n`); + } + const rawHits = receiptResult?.hits ?? core.query(kb, question, { topK }); + const hits = rawHits.map((hit) => ({ title: kb.headings?.[hit.blockId] || hit.source || `Block ${hit.blockId}`, source: hit.source || kb.docIds?.[hit.blockId] || 'unknown', score: Number(hit.score.toFixed(4)), @@ -669,7 +685,7 @@ async function cmdQuery(core, args) { hits.map((h, i) => ({ blockId: i, score: h.score, text: h.snippet, source: h.source })), { budget: 'small' } ); - console.log(JSON.stringify({ question, packPath, topK, hits, contextPatch: patch }, null, 2)); + console.log(JSON.stringify({ question, packPath, topK, hits, receipt: receiptResult?.receipt, contextPatch: patch }, null, 2)); return; } @@ -686,6 +702,77 @@ async function cmdQuery(core, args) { }); } +function parsePackCommandArgs(args, command) { + const positional = []; + const flags = {}; + for (let i = 0; i < args.length; i++) { + const arg = args[i]; + if (arg === '--out' || arg === '--to' || arg === '--pack') flags[arg.slice(2)] = args[++i]; + else if (arg.startsWith('--')) throw createError(`Unknown flag for ${command}: ${arg}`); + else positional.push(arg); + } + return { positional, flags }; +} + +async function cmdInspect(core, args) { + const { positional } = parsePackCommandArgs(args, 'inspect'); + const packPath = path.resolve(process.cwd(), positional[0] || 'dist/knowledge.knolo'); + if (!existsSync(packPath)) throw createError(`Pack file not found at ${path.relative(process.cwd(), packPath)}.`); + const bytes = Uint8Array.from(readFileSync(packPath)); + const pack = await mountPackFromBytes(core, bytes); + const result = { format: pack.meta?.format || (pack.meta?.version === 4 ? 'v4' : `v${pack.meta?.version ?? 'unknown'}`), version: pack.meta?.version, stats: pack.meta?.stats, blocks: pack.blocks.length, semantic: Boolean(pack.semantic), claims: Boolean(pack.claimGraph) }; + if (core.isPackV4?.(bytes.buffer)) result.container = core.inspectPackV4(bytes.buffer); + console.log(JSON.stringify(result, null, 2)); +} + +async function cmdVerify(core, args) { + const { positional } = parsePackCommandArgs(args, 'verify'); + const packPath = path.resolve(process.cwd(), positional[0] || 'dist/knowledge.knolo'); + if (!existsSync(packPath)) throw createError(`Pack file not found at ${path.relative(process.cwd(), packPath)}.`); + const bytes = Uint8Array.from(readFileSync(packPath)); + const pack = await mountPackFromBytes(core, bytes); + console.log(JSON.stringify({ verified: true, format: pack.meta?.format || (pack.meta?.version === 4 ? 'v4' : `v${pack.meta?.version}`), blocks: pack.blocks.length, terms: pack.meta?.stats?.terms ?? pack.lexicon.size }, null, 2)); +} + +async function cmdMigrate(core, args) { + const { positional, flags } = parsePackCommandArgs(args, 'migrate'); + const input = positional[0]; + if (!input || !flags.out) throw createError('Usage: knolo migrate --out --to 4'); + if (String(flags.to || '4') !== '4') throw createError('Only migration to format 4 is currently supported.'); + const inputPath = path.resolve(process.cwd(), input); + if (!existsSync(inputPath)) throw createError(`Pack file not found at ${path.relative(process.cwd(), inputPath)}.`); + const bytes = Uint8Array.from(readFileSync(inputPath)); + const pack = await mountPackFromBytes(core, bytes); + if (core.isPackV4?.(bytes.buffer)) throw createError('Input pack is already v4.'); + const docs = pack.blocks.map((text, i) => ({ id: pack.docIds?.[i] ?? undefined, heading: pack.headings?.[i] ?? undefined, namespace: pack.namespaces?.[i] ?? undefined, text })); + const migrated = core.serializePackV4(pack, docs); + const outPath = path.resolve(process.cwd(), flags.out); + mkdirSync(path.dirname(outPath), { recursive: true }); + writeFileSync(outPath, Buffer.from(migrated)); + console.log(`βœ” migrated ${path.relative(process.cwd(), inputPath)} to ${path.relative(process.cwd(), outPath)}`); +} + +async function cmdExplain(core, args) { + const { positional, flags } = parsePackCommandArgs(args, 'explain'); + const receiptPath = positional[0]; + if (!receiptPath || !flags.out && !flags.pack) throw createError('Usage: knolo explain --pack '); + const packPath = path.resolve(process.cwd(), flags.pack); + const receipt = JSON.parse(readFileSync(path.resolve(process.cwd(), receiptPath), 'utf8')); + const pack = await mountPackFromBytes(core, Uint8Array.from(readFileSync(packPath))); + core.verifyReceipt(receipt, pack); + console.log(JSON.stringify({ verified: true, decision: receipt.decision, answerability: receipt.answerability, reasons: receipt.reasons, query: receipt.normalizedQuery, hits: receipt.hits.map((hit) => ({ blockId: hit.blockId, score: hit.score, source: hit.source, spans: hit.spans })) }, null, 2)); +} + +async function cmdDiff(core, args) { + const { positional } = parsePackCommandArgs(args, 'diff'); + if (positional.length < 2) throw createError('Usage: knolo diff '); + const load = async (file) => { const bytes = Uint8Array.from(readFileSync(path.resolve(process.cwd(), file))); return { bytes, pack: await mountPackFromBytes(core, bytes) }; }; + const a = await load(positional[0]); const b = await load(positional[1]); + const aDigest = core.packDigest(a.pack); const bDigest = core.packDigest(b.pack); + const aSources = new Set(a.pack.docIds?.filter(Boolean) ?? []); const bSources = new Set(b.pack.docIds?.filter(Boolean) ?? []); + console.log(JSON.stringify({ equal: aDigest === bDigest, a: { packDigest: aDigest, blocks: a.pack.blocks.length, terms: a.pack.lexicon.size }, b: { packDigest: bDigest, blocks: b.pack.blocks.length, terms: b.pack.lexicon.size }, addedSources: [...bSources].filter((id) => !aSources.has(id)).sort(), removedSources: [...aSources].filter((id) => !bSources.has(id)).sort() }, null, 2)); +} + function parseKeyValueArgs(args) { const out = {}; for (let i = 0; i < args.length; i++) { @@ -1049,6 +1136,11 @@ async function main() { const core = await loadCore(); if (command === 'build') return await cmdBuild(core); if (command === 'query') return await cmdQuery(core, commandArgs); + if (command === 'inspect') return await cmdInspect(core, commandArgs); + if (command === 'migrate') return await cmdMigrate(core, commandArgs); + if (command === 'verify') return await cmdVerify(core, commandArgs); + if (command === 'explain') return await cmdExplain(core, commandArgs); + if (command === 'diff') return await cmdDiff(core, commandArgs); if (command === 'dev') return await cmdDev(core); if (command === 'semantic:index') return await cmdSemanticIndex(core, commandArgs); if (command === 'semantic:inspect') return await cmdSemanticInspect(core, commandArgs); diff --git a/packages/cli/package.json b/packages/cli/package.json index a29e774..48da3b4 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@knolo/cli", - "version": "3.2.2", + "version": "4.0.0", "type": "module", "bin": { "knolo": "bin/knolo.mjs" @@ -14,7 +14,7 @@ "test": "npm run build --prefix ../core && node --test test/cli.test.mjs" }, "dependencies": { - "@knolo/core": "^3.2.0" + "@knolo/core": "^4.0.0" }, "private": false } diff --git a/packages/cli/templates/icp-knowledge-canister/README.md b/packages/cli/templates/icp-knowledge-canister/README.md index b15969c..7ab41b3 100644 --- a/packages/cli/templates/icp-knowledge-canister/README.md +++ b/packages/cli/templates/icp-knowledge-canister/README.md @@ -65,3 +65,7 @@ bash scripts/query.sh "alpha beta" ## Sample Content The pack is built from the checked-in files under `knowledge/`, so the example stays deterministic and easy to rerun locally. +# Knolo v4 compatibility + +This template uses the ICP legacy v3 pack profile. The TypeScript CLI’s +`icp build-pack` command opts into v3 until the canister supports v4. diff --git a/packages/cli/test/cli.test.mjs b/packages/cli/test/cli.test.mjs index 9a62bcf..19e2311 100644 --- a/packages/cli/test/cli.test.mjs +++ b/packages/cli/test/cli.test.mjs @@ -154,6 +154,30 @@ test('build produces default pack', () => { assert.ok(existsSync(path.join(cwd, 'dist/knowledge.knolo'))); }); +test('inspect and verify expose the v4 container', () => { + const cwd = mkdtempSync(path.join(tmpdir(), 'knolo-cli-v4-inspect-')); + runCli(['init'], cwd); + runCli(['build'], cwd); + + const inspected = JSON.parse(runCli(['inspect', './dist/knowledge.knolo'], cwd)); + assert.equal(inspected.format, 'v4'); + assert.ok(inspected.container.sections.some((section) => section.name === 'manifest')); + + const verified = JSON.parse(runCli(['verify', './dist/knowledge.knolo'], cwd)); + assert.equal(verified.verified, true); +}); + +test('migrate converts an ICP-compatible legacy pack to v4', () => { + const cwd = mkdtempSync(path.join(tmpdir(), 'knolo-cli-v4-migrate-')); + const docsDir = path.join(cwd, 'docs'); + mkdirSync(docsDir); + writeFileSync(path.join(docsDir, 'alpha.txt'), 'alpha legacy text', 'utf8'); + runCli(['icp', 'build-pack', './docs', '--out', './legacy.knolo'], cwd); + runCli(['migrate', './legacy.knolo', '--out', './migrated.knolo', '--to', '4'], cwd); + const verified = JSON.parse(runCli(['verify', './migrated.knolo'], cwd)); + assert.equal(verified.format, 'v4'); +}); + test('query returns hit from sample doc', () => { const cwd = mkdtempSync(path.join(tmpdir(), 'knolo-cli-query-')); runCli(['init'], cwd); @@ -164,6 +188,16 @@ test('query returns hit from sample doc', () => { assert.match(output, /docs\/hello\.md/); }); +test('query receipts can be explained and verified', () => { + const cwd = mkdtempSync(path.join(tmpdir(), 'knolo-cli-receipt-')); + runCli(['init'], cwd); + runCli(['build'], cwd); + runCli(['query', 'hello', '--receipt', './receipt.json', '--json'], cwd); + assert.ok(existsSync(path.join(cwd, 'receipt.json'))); + const explained = runCli(['explain', './receipt.json', '--pack', './dist/knowledge.knolo'], cwd); + assert.match(explained, /"verified": true/); +}); + test('add updates existing source path', () => { const cwd = mkdtempSync(path.join(tmpdir(), 'knolo-cli-add-')); runCli(['init'], cwd); diff --git a/packages/core-python/README.md b/packages/core-python/README.md index 62814a5..94abe8c 100644 --- a/packages/core-python/README.md +++ b/packages/core-python/README.md @@ -1,8 +1,12 @@ -# `knolo` +# `knolo` β€” legacy v3-compatible runtime + +**Compatibility status:** legacy v1–v3 Python reader/query profile. The v4 +container, analyzer profiles, retrieval plans, receipts, and TrustBench +equivalence are currently implemented in TypeScript first. `knolo` is the pure-Python runtime for mounting existing `.knolo` packs and running deterministic lexical queries locally. -It is intentionally release-scoped for Phase 2: +It is intentionally release-scoped for the legacy compatibility profile: - local-first retrieval - deterministic lexical retrieval @@ -11,6 +15,7 @@ It is intentionally release-scoped for Phase 2: - no Python pack builder - no LangChain or LlamaIndex integration - no Node.js runtime dependency for mount/query +- not certified as v4-equivalent in `conformance/` Packs are still built with `@knolo/core` in TypeScript, then mounted and queried from Python. diff --git a/packages/core-python/RELEASE.md b/packages/core-python/RELEASE.md index 6faaf9b..e05d1a5 100644 --- a/packages/core-python/RELEASE.md +++ b/packages/core-python/RELEASE.md @@ -11,3 +11,8 @@ - [ ] Confirm the publish workflow only runs on GitHub release publication and uses Trusted Publishing with no hardcoded secrets. - [ ] Smoke install the built wheel in a clean environment and run a basic `mount_pack` / `query` check. - [ ] Yank a bad PyPI release instead of republishing the same tag if a release needs to be rolled back. +# Phase 5 compatibility note + +The Python runtime remains a legacy v1–v3 reader/query profile. It is not +marked equivalent in the shared Phase 5 TrustBench output until it supports +the v4 container, analyzer profile digest, retrieval plan, and receipt contract. diff --git a/packages/core-python/tests/fixtures/README.md b/packages/core-python/tests/fixtures/README.md index ba8b0bd..6507c80 100644 --- a/packages/core-python/tests/fixtures/README.md +++ b/packages/core-python/tests/fixtures/README.md @@ -19,3 +19,7 @@ node scripts/regenerate-python-fixture.mjs ``` Pass `--check` to verify that the working tree bytes still match the corpus without rewriting the file. +# Legacy compatibility fixture + +This fixture intentionally remains v3 for the Python runtime compatibility +profile. It is not the Phase 5 v4 TrustBench reference pack. diff --git a/packages/core-rust/README.md b/packages/core-rust/README.md index e9e5a1e..9bb5980 100644 --- a/packages/core-rust/README.md +++ b/packages/core-rust/README.md @@ -1,5 +1,9 @@ # knolo-core-rust +**Compatibility status:** legacy v1–v3 Rust reader/query profile. The +TypeScript runtime is the Phase 5 v4 TrustBench reference until Rust supports +the v4 container, analyzer identity, retrieval plan, and receipt contract. + Native Rust runtime support for Knolo `.knolo` packs. ## Included in this initial release diff --git a/packages/core/CHANGELOG.md b/packages/core/CHANGELOG.md index a4a0a7a..3b4c8f4 100644 --- a/packages/core/CHANGELOG.md +++ b/packages/core/CHANGELOG.md @@ -4,7 +4,48 @@ All notable changes to this project will be documented in this file. ## [Unreleased] +### v4.0.0 / Phase 5 +- Completed the TypeScript TrustBench reference profile and repository conformance documentation. +- Added aggregate Recall@K, MRR, nDCG, hit-count, abstention-precision, and receipt-verification reporting. +- Bumped `@knolo/core` to `4.0.0`; legacy v1–v3 readers remain supported. + +### Changed +- Enforced namespace, source, and required-phrase constraints after every lexical, expansion, and reranking stage. +- Removed KNS relevance perturbations; equal scores now use stable block-ID ordering. +- Preserved raw source text in built packs so Markdown, code, paths, identifiers, and evidence spans remain intact. +- Phase 2 packs now use a self-describing v4 container by default, with SHA-256 manifest, section, and pack identities. +- Legacy v1–v3 packs remain readable; `buildPack(..., { format: 3 })` preserves the legacy writer for older runtimes. + ### Added +- Strict v4 header, section-directory, bounds, overlap, schema, and digest validation. +- v4 source manifests and stable document chunks with exact raw spans. +- CLI support for `knolo inspect`, `knolo migrate --to 4`, and `knolo verify`. +- Browser/React-Native-safe SHA-256 implementation for independent pack verification. +- Phase 3 analyzer profiles, profile digests, fielded chunk signals, and deterministic `queryWithPlan()` retrieval plans. +- Phase 4 `queryWithReceipt()`/`verifyReceipt()` APIs with pack/source identities, exact evidence spans, answer/clarify/abstain decisions, and replay hashes. +- Phase 5 TrustBench conformance fixtures, canonical retrieval output, corruption fixtures, and reproducible Recall@K/MRR evaluation. + +### Phase 2 handoff +- Phase 3 should make the v4 analyzer profile and retrieval plan canonical; the current v4 chunks expose raw text and basic source spans but do not yet implement fielded retrieval. +- Phase 4 should bind receipts to the v4 manifest/section digests and add independent receipt verification. +- ICP and other external runtimes should add v4 readers before their builders switch from `format: 3`. + +### Phase 3 handoff +- Phase 4 should add receipts around `queryWithPlan()` and bind plan hashes to manifest/source digests. +- Fielded retrieval currently uses deterministic chunk metadata for heading, code-symbol, path, and table signals; future work can split these into dedicated postings channels without changing the plan contract. +- Analyzer profiles are immutable declarations with SHA-256 digests; new profiles require explicit IDs and conformance fixtures. + +### Phase 4 handoff +- Receipts are integrity-verifiable but unsigned; Phase 6 should add key IDs, signatures, rotation, revocation, and signed quality certificates. +- ClaimGraph edges remain deterministic and evidence-indexed by block IDs; Phase 5/next graph work should add source digests, temporal validity, authority, contradiction, and conflict policies. +- `query --receipt --json`, `explain`, and `diff` provide local operator workflows; hosted registry/distribution is intentionally out of scope. + +### Phase 5 handoff +- `conformance/` is the shared fixture contract; regenerate with `npm run trustbench:generate` and gate with `npm run trustbench:test`. +- TypeScript is the first canonical runtime profile. Rust, Python, and ICP remain explicitly unmarked until they consume the v4 fixtures and produce equivalent canonical rows. +- Metrics currently include Recall@K, reciprocal rank, hit count, decision, receipt verification, and deterministic plan hashes. TrustBench can add nDCG, attribution, abstention calibration, and latency baselines as runtime profiles mature. + +### Earlier additions - Deterministic append-only document patch packs with base fingerprints, JSON serialization, merging, replay into `LivePack`, and `LivePack.serializePatchPack()`. - Knolo Cortex, a local-first overlay memory layer for `.knolo` packs with deterministic lexical recall, append-only logs, portable serialization, and no required vector DB. - Added the initial memory surface under `@knolo/core`, including memory normalization, immutable cortex writes, recall ranking, and consolidation helpers, while keeping the existing pack runtime API unchanged. diff --git a/packages/core/README.md b/packages/core/README.md index 177dd0f..465f0db 100644 --- a/packages/core/README.md +++ b/packages/core/README.md @@ -1,5 +1,9 @@ # πŸ“¦ `@knolo/core` +**Version 4 contract:** v4 packs, analyzer profiles, deterministic retrieval +plans, verifiable receipts, evidence spans, and TrustBench conformance are the +current TypeScript runtime surface. + `@knolo/core` is the **deterministic retrieval engine and pack runtime** behind Knolo. It lets you: @@ -139,6 +143,9 @@ const bytes = await buildPack(docs, { `buildPack` produces a versioned `.knolo` binary artifact. +It emits a v4 container by default. Use `{ format: 3 }` only when targeting a +legacy runtime. v1–v3 packs remain readable. + You can write it to disk or store it in object storage. --- diff --git a/packages/core/package.json b/packages/core/package.json index 928dd3d..cce293a 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -1,6 +1,6 @@ { "name": "@knolo/core", - "version": "3.5.0", + "version": "4.0.0", "type": "module", "description": "Local-first knowledge packs for small LLMs.", "keywords": [ diff --git a/packages/core/scripts/test.mjs b/packages/core/scripts/test.mjs index 286044a..4af7d85 100644 --- a/packages/core/scripts/test.mjs +++ b/packages/core/scripts/test.mjs @@ -9,6 +9,11 @@ import { buildPack, mountPack, query, + queryWithPlan, + queryWithReceipt, + verifyReceipt, + ANALYZER_PROFILES, + analyzerProfileDigest, makeContextPatch, decodeScaleF16, lexConfidence, @@ -348,6 +353,64 @@ async function testQueryExpansionRecall() { ); } +async function testHardConstraintsSurviveQueryExpansion() { + const docs = [ + { + id: 'allowed', + namespace: 'public', + text: 'Throttling controls event bursts and smooths bridge pressure.', + }, + { + id: 'blocked', + namespace: 'private', + text: 'Rate limiting caps request bursts and protects systems under load.', + }, + ]; + const pack = await mountPack({ src: await buildPack(docs) }); + const hits = query(pack, 'throttling bridge pressure', { + topK: 5, + namespace: 'public', + queryExpansion: { enabled: true, docs: 2, terms: 4, weight: 1 }, + }); + assert.ok(hits.length > 0, 'expected the scoped seed document to remain searchable'); + assert.ok(hits.every((hit) => hit.namespace === 'public'), 'expansion must not widen namespace scope'); + assert.ok(!hits.some((hit) => hit.source === 'blocked'), 'expansion must not reintroduce blocked sources'); +} + +async function testSourceTextIsPreserved() { + const source = '# API\n\n```ts\nconst path = `a/b`;\n```\n\nUse [the path](https://example.test).'; + const pack = await mountPack({ src: await buildPack([{ id: 'raw', text: source }]) }); + assert.equal(pack.blocks[0], source, 'pack blocks must preserve raw source text for evidence'); +} + +async function testPhase3AnalyzerAndRetrievalPlan() { + const docs = [{ id: 'code', heading: 'TypeScript paths', namespace: 'docs', text: '# TypeScript paths\n\n```ts\nconst resolver = "./src/index.ts";\n```' }]; + const pack = await mountPack({ src: await buildPack(docs, { analyzer: 'knolo-analyzer/code-typescript-v1' }) }); + assert.equal(pack.meta.analyzer.id, 'knolo-analyzer/code-typescript-v1'); + assert.equal(pack.meta.analyzer.digest, analyzerProfileDigest(ANALYZER_PROFILES['knolo-analyzer/code-typescript-v1'])); + assert.ok(pack.chunks?.[0]?.codeSymbols?.includes('resolver')); + assert.ok(pack.chunks?.[0]?.paths?.includes('./src/index.ts')); + + const first = queryWithPlan(pack, 'resolver', { topK: 3 }); + const second = queryWithPlan(pack, 'resolver', { topK: 3 }); + assert.equal(first.plan.version, 'retrieval-v4.0'); + assert.equal(first.plan.planHash, second.plan.planHash); + assert.equal(first.hits[0]?.evidence?.planHash, first.plan.planHash); +} + +async function testPhase4ReceiptVerification() { + const pack = await mountPack({ src: await buildPack([{ id: 'receipt-doc', text: 'Receipt verification requires exact evidence.' }]) }); + const result = queryWithReceipt(pack, 'receipt verification', { topK: 2, policy: { minAnswerability: 0.1 } }); + assert.equal(result.receipt.version, 'receipt-v1'); + assert.equal(result.receipt.decision, 'answer'); + assert.ok(result.receipt.replayHash.startsWith('sha256-')); + assert.ok(result.receipt.hits[0]?.spans[0]?.text); + verifyReceipt(result.receipt, pack); + const tampered = structuredClone(result.receipt); + tampered.hits[0].spans[0].text = 'tampered'; + assert.throws(() => verifyReceipt(tampered, pack), /evidence span mismatch|replay hash mismatch/); +} + async function testMinScoreFiltering() { const docs = [ { @@ -722,7 +785,7 @@ async function testSemanticEvidenceScoresRemainCorrectAfterRerank() { enabled: true, queryEmbedding: new Float32Array([0, 1]), force: true, - blend: { enabled: true, wLex: 0.5, wSem: 0.5 }, + blend: { enabled: true, wLex: 0.4, wSem: 0.6 }, }, }); @@ -1806,6 +1869,10 @@ await testPackStoresOneBasedTokenPositions(); await testNearDuplicateDedupe(); await testNamespaceFiltering(); await testQueryExpansionRecall(); +await testHardConstraintsSurviveQueryExpansion(); +await testSourceTextIsPreserved(); +await testPhase3AnalyzerAndRetrievalPlan(); +await testPhase4ReceiptVerification(); await testSourceFiltering(); await testMinScoreFiltering(); await testContextPatchSourcePropagation(); diff --git a/packages/core/src/analyzer.ts b/packages/core/src/analyzer.ts new file mode 100644 index 0000000..7ebea5c --- /dev/null +++ b/packages/core/src/analyzer.ts @@ -0,0 +1,25 @@ +import { getTextEncoder } from './utils/utf8.js'; +import { sha256Hex } from './utils/sha256.js'; + +export type AnalyzerProfile = { + id: 'knolo-analyzer/en-technical-v1' | 'knolo-analyzer/code-typescript-v1' | 'knolo-analyzer/multilingual-general-v1'; + version: 1; + unicode: 'nfkd-diacritic-fold'; + identifier: 'split-punctuation-preserve-hyphen'; + exactToken: boolean; + pathTokens: boolean; +}; + +export const ANALYZER_PROFILES: Record = { + 'knolo-analyzer/en-technical-v1': { id: 'knolo-analyzer/en-technical-v1', version: 1, unicode: 'nfkd-diacritic-fold', identifier: 'split-punctuation-preserve-hyphen', exactToken: true, pathTokens: true }, + 'knolo-analyzer/code-typescript-v1': { id: 'knolo-analyzer/code-typescript-v1', version: 1, unicode: 'nfkd-diacritic-fold', identifier: 'split-punctuation-preserve-hyphen', exactToken: true, pathTokens: true }, + 'knolo-analyzer/multilingual-general-v1': { id: 'knolo-analyzer/multilingual-general-v1', version: 1, unicode: 'nfkd-diacritic-fold', identifier: 'split-punctuation-preserve-hyphen', exactToken: true, pathTokens: true }, +}; + +export function resolveAnalyzerProfile(id?: AnalyzerProfile['id']): AnalyzerProfile { + return ANALYZER_PROFILES[id ?? 'knolo-analyzer/en-technical-v1']; +} + +export function analyzerProfileDigest(profile: AnalyzerProfile): string { + return `sha256-${sha256Hex(getTextEncoder().encode(JSON.stringify(profile)))}`; +} diff --git a/packages/core/src/builder.ts b/packages/core/src/builder.ts index 39fc03b..6e87e6a 100644 --- a/packages/core/src/builder.ts +++ b/packages/core/src/builder.ts @@ -1,7 +1,7 @@ /* * builder.ts * - * Build `.knolo` packs from input docs. Persists headings/docIds/token lengths + * Build `.knolo` packs from input docs. Persists raw source text, headings/docIds/token lengths * and stores avgBlockLen in meta for stable query-time normalization. */ @@ -13,6 +13,9 @@ import { encodeScaleF16, quantizeEmbeddingInt8L2Norm } from './semantic.js'; import type { AgentDefinitionV1, AgentRegistry } from './agent.js'; import { validateAgentRegistry } from './agent.js'; import { buildClaimGraph } from './graph/build_claim_graph.js'; +import { mountPackFromBuffer } from './pack.runtime.js'; +import { serializePackV4 } from './pack.v4.js'; +import { analyzerProfileDigest, resolveAnalyzerProfile, type AnalyzerProfile } from './analyzer.js'; export type BuildInputDoc = { id?: string; @@ -21,6 +24,9 @@ export type BuildInputDoc = { text: string; }; export type BuildPackOptions = { + /** Defaults to v4. Use 3 only for legacy runtimes such as older ICP canisters. */ + format?: 3 | 4; + analyzer?: AnalyzerProfile['id']; agents?: AgentRegistry | AgentDefinitionV1[]; semantic?: { enabled: boolean; @@ -40,10 +46,11 @@ export async function buildPack( ): Promise { const normalizedDocs = validateDocs(docs); - // Prepare blocks (strip MD) and carry heading/docId for optional boosts. + // Preserve source text exactly. Tokenization is an indexing concern; the + // stored block is also the evidence/citation surface returned to callers. const blocks: Block[] = normalizedDocs.map((d, i) => ({ id: i, - text: stripMd(d.text), + text: d.text, heading: d.heading, })); @@ -60,8 +67,10 @@ export async function buildPack( ? buildClaimGraph(normalizedDocs, { maxEdgesPerDoc: opts.graph?.maxEdgesPerDoc }) : null; + const analyzer = resolveAnalyzerProfile(opts.analyzer); const meta = { version: 3, + analyzer: { id: analyzer.id, version: analyzer.version, digest: analyzerProfileDigest(analyzer) }, stats: { docs: normalizedDocs.length, blocks: blocks.length, @@ -167,7 +176,8 @@ export async function buildPack( out.set(graphBytes, offset); } - return out; + if (opts.format === 3) return out; + return serializePackV4(mountPackFromBuffer(out.buffer), normalizedDocs); } function normalizeAgents( @@ -294,15 +304,3 @@ function validateDocs(docs: BuildInputDoc[]): BuildInputDoc[] { return doc; }); } - -/** Strip Markdown syntax with lightweight regexes (no deps). */ -function stripMd(md: string): string { - let text = md.replace(/```[\s\S]*?```/g, ' '); - text = text.replace(/`[^`]*`/g, ' '); - text = text.replace(/[\*_~]+/g, ' '); - text = text.replace(/^#+\s*/gm, ''); - text = text.replace(/\[([^\]]+)\]\([^)]+\)/g, '$1'); - text = text.replace(/[\[\]()]/g, ' '); - text = text.replace(/\s+/g, ' ').trim(); - return text; -} diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index b9fbd1f..60438e6 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -1,11 +1,18 @@ // src/index.ts export { mountPack, hasSemantic } from './pack.runtime.js'; +export { mountPackFromBuffer } from './pack.runtime.js'; +export { isPackV4, inspectPackV4, parsePackV4, serializePackV4, PACK_V4_MAGIC } from './pack.v4.js'; +export { ANALYZER_PROFILES, analyzerProfileDigest, resolveAnalyzerProfile } from './analyzer.js'; export { query, lexConfidence, validateQueryOptions, validateSemanticQueryOptions, + applyHardConstraints, + queryWithPlan, } from './query.js'; +export { createRetrievalPlan } from './retrieval_plan.js'; +export { queryWithReceipt, verifyReceipt, packDigest } from './receipt.js'; export { makeContextPatch } from './patch.js'; export { buildPack } from './builder.js'; export { LivePack, createLivePack } from './live.js'; @@ -56,13 +63,16 @@ export { } from './graph/log.js'; export { expandQueryWithGraph } from './graph/query_expand.js'; export * from './memory/index.js'; -export type { MountOptions, PackMeta, Pack } from './pack.runtime.js'; -export type { QueryOptions, Hit } from './query.js'; +export type { MountOptions, PackMeta, Pack, PackChunk } from './pack.runtime.js'; +export type { QueryOptions, Hit, QueryWithPlanResult } from './query.js'; +export type { RetrievalPlan } from './retrieval_plan.js'; +export type { QueryReceipt, ReceiptOptions, ReceiptDecision, EvidenceSpan } from './receipt.js'; export type { LivePackOptions } from './live.js'; export type { PatchPack, PatchPackV1, PatchOpV1, PatchUpsertV1, PatchRemoveV1 } from './patch_pack.js'; export type { EmbeddingProvider, SemanticSidecar, SemanticQueryOptions, RetrievalEvidence } from './semantic/types.js'; export type { ContextPatch } from './patch.js'; export type { BuildInputDoc, BuildPackOptions } from './builder.js'; +export type { AnalyzerProfile } from './analyzer.js'; export type { AgentPromptTemplate, AgentToolPolicy, diff --git a/packages/core/src/pack.runtime.ts b/packages/core/src/pack.runtime.ts index 9ebce0b..c3f8f05 100644 --- a/packages/core/src/pack.runtime.ts +++ b/packages/core/src/pack.runtime.ts @@ -10,6 +10,7 @@ import type { AgentRegistry } from './agent.js'; import { validateAgentRegistry } from './agent.js'; import type { ClaimGraph } from './graph/claim_graph.js'; import { validateClaimGraph } from './graph/claim_graph.js'; +import { isPackV4, parsePackV4 } from './pack.v4.js'; export type MountOptions = { src: string | ArrayBufferLike | Uint8Array }; @@ -18,6 +19,27 @@ export type PackMeta = { stats: { docs: number; blocks: number; terms: number; avgBlockLen?: number }; agents?: AgentRegistry; claimGraph?: { version: 1; nodes: number; edges: number }; + format?: 'legacy' | 'v4'; + analyzer?: { id: string; digest: string; version: number }; + manifestDigest?: string; + packDigest?: string; +}; + +export type PackChunk = { + id: number; + parentId?: number; + text: string; + displayText?: string; + fieldedText?: string; + heading?: string | null; + docId?: string | null; + namespace?: string | null; + len?: number; + span?: { start: number; end: number; lineStart?: number; lineEnd?: number }; + kind?: 'document' | 'markdown' | 'code' | 'json' | 'table'; + codeSymbols?: string[]; + paths?: string[]; + tableHeaders?: string[]; }; export type Pack = { @@ -29,6 +51,7 @@ export type Pack = { docIds?: (string | null)[]; namespaces?: (string | null)[]; blockTokenLens?: number[]; + chunks?: PackChunk[]; semantic?: { version: 1; modelId: string; @@ -53,27 +76,31 @@ export async function mountPack(opts: MountOptions): Promise { } export function mountPackFromBuffer(buf: ArrayBuffer): Pack { + if (isPackV4(buf)) return parsePackV4(buf); + if (buf.byteLength < 16) throw new Error('Invalid or truncated pack: missing legacy header.'); const dv = new DataView(buf); const dec = getTextDecoder(); let offset = 0; - const metaLen = dv.getUint32(offset, true); + const metaLen = readLength(dv, offset, buf.byteLength, 'metadata'); offset += 4; const metaJson = dec.decode(new Uint8Array(buf, offset, metaLen)); offset += metaLen; const meta: PackMeta = JSON.parse(metaJson); + if (!meta || !Number.isInteger(meta.version) || meta.version < 1 || meta.version > 3) throw new Error(`Unsupported legacy pack version: ${String(meta?.version)}.`); if (meta.agents) { validateAgentRegistry(meta.agents); } - const lexLen = dv.getUint32(offset, true); + const lexLen = readLength(dv, offset, buf.byteLength, 'lexicon'); offset += 4; const lexJson = dec.decode(new Uint8Array(buf, offset, lexLen)); offset += lexLen; const lexEntries: Array<[string, number]> = JSON.parse(lexJson); const lexicon = new Map(lexEntries); - const postCount = dv.getUint32(offset, true); + const postCount = readLength(dv, offset, buf.byteLength, 'postings'); + if (postCount > 128 * 1024 * 1024) throw new Error('Legacy postings declaration exceeds safety limit.'); offset += 4; const postings = new Uint32Array(postCount); for (let i = 0; i < postCount; i++) { @@ -81,7 +108,7 @@ export function mountPackFromBuffer(buf: ArrayBuffer): Pack { offset += 4; } - const blocksLen = dv.getUint32(offset, true); + const blocksLen = readLength(dv, offset, buf.byteLength, 'blocks'); offset += 4; const blocksJson = dec.decode(new Uint8Array(buf, offset, blocksLen)); offset += blocksLen; @@ -168,6 +195,7 @@ export function mountPackFromBuffer(buf: ArrayBuffer): Pack { break; } + if (blocks.length > 10_000_000 || (meta.stats?.blocks ?? blocks.length) !== blocks.length) throw new Error('Invalid legacy pack block count.'); return { meta, lexicon, @@ -182,6 +210,13 @@ export function mountPackFromBuffer(buf: ArrayBuffer): Pack { }; } +function readLength(dv: DataView, offset: number, total: number, label: string): number { + if (offset < 0 || offset + 4 > total) throw new Error(`Invalid or truncated pack: missing ${label} length.`); + const length = dv.getUint32(offset, true); + if (length > total - offset - 4) throw new Error(`Invalid or truncated pack: ${label} exceeds payload.`); + return length; +} + function looksLikeSemanticJson(parsed: unknown): boolean { if (!parsed || typeof parsed !== 'object') return false; const sem = parsed as { diff --git a/packages/core/src/pack.ts b/packages/core/src/pack.ts index 6082dc2..afcf503 100644 --- a/packages/core/src/pack.ts +++ b/packages/core/src/pack.ts @@ -1,2 +1,3 @@ export { mountPack, hasSemantic } from './pack.runtime.js'; -export type { MountOptions, PackMeta, Pack } from './pack.runtime.js'; +export type { MountOptions, PackMeta, Pack, PackChunk } from './pack.runtime.js'; +export { isPackV4, inspectPackV4, parsePackV4, serializePackV4, PACK_V4_MAGIC } from './pack.v4.js'; diff --git a/packages/core/src/pack.v4.ts b/packages/core/src/pack.v4.ts new file mode 100644 index 0000000..c726d44 --- /dev/null +++ b/packages/core/src/pack.v4.ts @@ -0,0 +1,144 @@ +import type { Pack, PackChunk } from './pack.runtime.js'; +import type { BuildInputDoc } from './builder.js'; +import { getTextDecoder, getTextEncoder } from './utils/utf8.js'; +import { sha256Hex } from './utils/sha256.js'; +import { validateClaimGraph } from './graph/claim_graph.js'; +import { analyzerProfileDigest, resolveAnalyzerProfile } from './analyzer.js'; + +export const PACK_V4_MAGIC = 'KNLOV4\0\0'; +const DIGEST_FIELD = 80; +const FIXED_HEADER = 32 + DIGEST_FIELD * 2; +const MAX_HEADER = 16 * 1024 * 1024; +const MAX_SECTION = 512 * 1024 * 1024; + +type Section = { name: string; schema: number; encoding: 'json' | 'raw'; required: boolean; capabilities: string[]; bytes: Uint8Array }; +type DirectoryEntry = Omit & { offset: number; length: number; digest: string }; + +export type PackManifestV4 = { + version: 4; + sources: Array<{ id: string; digest: string; mediaType: string; namespace?: string; parserVersion: string; analyzerVersion: string; chunking: { strategy: string; version: string } }>; +}; + +export function serializePackV4(pack: Pack, docs: BuildInputDoc[]): Uint8Array { + const enc = getTextEncoder(); + const manifest: PackManifestV4 = { version: 4, sources: docs.map((doc, i) => ({ + id: doc.id ?? `block-${i}`, + digest: `sha256-${sha256Hex(enc.encode(doc.text))}`, + mediaType: 'text/plain', + ...(doc.namespace ? { namespace: doc.namespace } : {}), + parserVersion: 'knolo-raw-v1', analyzerVersion: `${resolveAnalyzerProfile(pack.meta.analyzer?.id as any).id}@${analyzerProfileDigest(resolveAnalyzerProfile(pack.meta.analyzer?.id as any))}`, + chunking: { strategy: 'document', version: '1' }, + })) }; + const sections: Section[] = [ + { name: 'metadata', schema: 1, encoding: 'json', required: true, capabilities: [], bytes: enc.encode(JSON.stringify({ ...pack.meta, version: 4, format: 'v4' })) }, + { name: 'lexicon', schema: 1, encoding: 'json', required: true, capabilities: [], bytes: enc.encode(JSON.stringify([...pack.lexicon.entries()])) }, + { name: 'postings', schema: 1, encoding: 'raw', required: true, capabilities: [], bytes: u32Bytes(pack.postings) }, + { name: 'chunks', schema: 1, encoding: 'json', required: true, capabilities: [], bytes: enc.encode(JSON.stringify(pack.blocks.map((text, i) => analyzeChunk(text, i, pack)))) }, + { name: 'manifest', schema: 1, encoding: 'json', required: true, capabilities: [], bytes: enc.encode(JSON.stringify(manifest)) }, + ]; + if (pack.semantic) { + sections.push({ name: 'semantic', schema: 1, encoding: 'json', required: false, capabilities: ['semantic'], bytes: enc.encode(JSON.stringify({ ...pack.semantic, vecs: Array.from(pack.semantic.vecs), scales: pack.semantic.scales ? Array.from(pack.semantic.scales) : undefined })) }); + } + if (pack.claimGraph) sections.push({ name: 'claims', schema: 1, encoding: 'json', required: false, capabilities: ['claims'], bytes: enc.encode(JSON.stringify(pack.claimGraph)) }); + + const manifestDigest = `sha256-${sha256Hex(sections.find((s) => s.name === 'manifest')!.bytes)}`; + let directory: DirectoryEntry[] = []; + let headerLength = FIXED_HEADER; + for (let attempt = 0; attempt < 4; attempt++) { + let offset = headerLength; + directory = sections.map((s) => { const entry = { name: s.name, schema: s.schema, encoding: s.encoding, required: s.required, capabilities: s.capabilities, offset, length: s.bytes.byteLength, digest: `sha256-${sha256Hex(s.bytes)}` }; offset += s.bytes.byteLength; return entry; }); + const dirBytes = enc.encode(JSON.stringify(directory)); + const next = FIXED_HEADER + enc.encode(JSON.stringify(manifest)).byteLength + dirBytes.byteLength; + if (next === headerLength) break; + headerLength = next; + } + const dirBytes = enc.encode(JSON.stringify(directory)); + const manifestBytes = enc.encode(JSON.stringify(manifest)); + const sectionBytes = sections.reduce((n, s) => n + s.bytes.byteLength, 0); + const total = headerLength + sectionBytes; + const out = new Uint8Array(total); + const dv = new DataView(out.buffer); + for (let i = 0; i < 8; i++) out[i] = PACK_V4_MAGIC.charCodeAt(i); + dv.setUint16(8, 4, true); dv.setUint16(10, 0, true); dv.setUint32(12, headerLength, true); + dv.setUint32(16, sections.length, true); dv.setUint32(20, manifestBytes.byteLength, true); dv.setUint32(24, dirBytes.byteLength, true); dv.setUint32(28, 0, true); + let at = FIXED_HEADER; out.set(manifestBytes, at); at += manifestBytes.byteLength; out.set(dirBytes, at); at += dirBytes.byteLength; + for (const section of sections) { out.set(section.bytes, at); at += section.bytes.byteLength; } + const packDigest = `sha256-${sha256Hex(out.slice(headerLength))}`; + writeAscii(out, 32, manifestDigest, DIGEST_FIELD); + writeAscii(out, 32 + DIGEST_FIELD, packDigest, DIGEST_FIELD); + return out; +} + +export function isPackV4(buf: ArrayBuffer): boolean { + if (buf.byteLength < 8) return false; + const bytes = new Uint8Array(buf, 0, 8); + return String.fromCharCode(...bytes) === PACK_V4_MAGIC; +} + +export function inspectPackV4(buf: ArrayBuffer): { format: 4; version: number; flags: number; sections: Array<{ name: string; schema: number; encoding: string; required: boolean; offset: number; length: number; digest: string }>; manifest: PackManifestV4; manifestDigest: string; packDigest: string } { + if (!isPackV4(buf)) throw new Error('Not a v4 pack.'); + const dv = new DataView(buf); const dec = getTextDecoder(); + const headerLength = dv.getUint32(12, true), manifestLength = dv.getUint32(20, true), directoryLength = dv.getUint32(24, true); + const manifest = JSON.parse(dec.decode(new Uint8Array(buf, FIXED_HEADER, manifestLength))) as PackManifestV4; + const sections = JSON.parse(dec.decode(new Uint8Array(buf, FIXED_HEADER + manifestLength, directoryLength))) as DirectoryEntry[]; + return { format: 4, version: dv.getUint16(8, true), flags: dv.getUint16(10, true), sections, manifest, manifestDigest: readAscii(new Uint8Array(buf), 32, DIGEST_FIELD), packDigest: readAscii(new Uint8Array(buf), 32 + DIGEST_FIELD, DIGEST_FIELD) }; +} + +export function parsePackV4(buf: ArrayBuffer): Pack { + const dv = new DataView(buf); const dec = getTextDecoder(); + if (dv.getUint16(8, true) !== 4) throw new Error('Unsupported pack format version.'); + const headerLength = dv.getUint32(12, true), sectionCount = dv.getUint32(16, true), manifestLength = dv.getUint32(20, true), directoryLength = dv.getUint32(24, true); + if (headerLength < FIXED_HEADER || headerLength > MAX_HEADER || sectionCount < 1 || sectionCount > 1024 || manifestLength > headerLength || directoryLength > headerLength) throw new Error('Invalid v4 pack header.'); + const manifestStart = FIXED_HEADER, dirStart = manifestStart + manifestLength; + if (dirStart + directoryLength !== headerLength || headerLength > buf.byteLength) throw new Error('Invalid v4 pack header length.'); + const manifestDigest = readAscii(new Uint8Array(buf), 32, DIGEST_FIELD); + if (manifestDigest !== `sha256-${sha256Hex(new Uint8Array(buf, manifestStart, manifestLength))}`) throw new Error('v4 manifest digest mismatch.'); + let manifest: PackManifestV4; let directory: DirectoryEntry[]; + try { manifest = JSON.parse(dec.decode(new Uint8Array(buf, manifestStart, manifestLength))); directory = JSON.parse(dec.decode(new Uint8Array(buf, dirStart, directoryLength))); } catch { throw new Error('Invalid v4 pack manifest or section directory.'); } + if (manifest.version !== 4 || !Array.isArray(directory) || directory.length !== sectionCount) throw new Error('Invalid v4 pack manifest or section directory.'); + const sections = new Map(); + const ranges: Array<{ start: number; end: number; name: string }> = []; + for (const entry of directory) { + if (!entry || typeof entry.name !== 'string' || entry.offset < headerLength || entry.length < 0 || entry.length > MAX_SECTION || entry.offset + entry.length > buf.byteLength || sections.has(entry.name)) throw new Error(`Invalid v4 section: ${String(entry?.name ?? '')}.`); + const bytes = new Uint8Array(buf, entry.offset, entry.length); + ranges.push({ start: entry.offset, end: entry.offset + entry.length, name: entry.name }); + if (entry.digest !== `sha256-${sha256Hex(bytes)}`) throw new Error(`v4 section digest mismatch: ${entry.name}.`); + sections.set(entry.name, bytes); + } + ranges.sort((a, b) => a.start - b.start); + for (let i = 1; i < ranges.length; i++) if (ranges[i].start < ranges[i - 1].end) throw new Error(`Overlapping v4 sections: ${ranges[i - 1].name} and ${ranges[i].name}.`); + const storedPackDigest = readAscii(new Uint8Array(buf), 32 + DIGEST_FIELD, DIGEST_FIELD); + const sectionEnd = ranges.reduce((end, range) => Math.max(end, range.end), headerLength); + if (storedPackDigest !== `sha256-${sha256Hex(new Uint8Array(buf, headerLength, sectionEnd - headerLength))}`) throw new Error('v4 pack digest mismatch.'); + for (const required of ['metadata','lexicon','postings','chunks','manifest']) if (!sections.has(required)) throw new Error(`Missing required v4 section: ${required}.`); + const json = (name: string) => JSON.parse(dec.decode(sections.get(name)!)); + const meta = json('metadata'); const entries = json('lexicon'); const chunks = json('chunks'); + if (!meta || meta.version !== 4 || !Array.isArray(entries) || !Array.isArray(chunks)) throw new Error('Invalid v4 required section schema.'); + const postingsBytes = sections.get('postings')!; if (postingsBytes.byteLength % 4) throw new Error('Invalid v4 postings section length.'); + const postings = new Uint32Array(postingsBytes.byteLength / 4); const pdv = new DataView(postingsBytes.buffer, postingsBytes.byteOffset, postingsBytes.byteLength); for (let i=0;i String(x?.text ?? '')); const headings = chunks.map((x: any) => x?.heading ?? null); const docIds = chunks.map((x: any) => x?.docId ?? null); const namespaces = chunks.map((x: any) => x?.namespace ?? null); const blockTokenLens = chunks.map((x: any) => Number(x?.len ?? 0)); + if (blocks.length !== meta.stats?.blocks || entries.some((x: any) => !Array.isArray(x) || typeof x[0] !== 'string' || !Number.isInteger(x[1]) || x[1] < 1)) throw new Error('Invalid v4 chunk or lexicon schema.'); + let semantic: Pack['semantic']; + if (sections.has('semantic')) { + const value = json('semantic'); + if (value?.encoding !== 'int8_l2norm' || !Array.isArray(value.vecs) || !Number.isInteger(value.dims) || value.dims < 1) throw new Error('Invalid v4 semantic section schema.'); + semantic = { version: 1, modelId: String(value.modelId ?? ''), dims: value.dims, encoding: 'int8_l2norm', perVectorScale: Boolean(value.perVectorScale), vecs: Int8Array.from(value.vecs), scales: Array.isArray(value.scales) ? Uint16Array.from(value.scales) : undefined }; + } + const claimGraph = sections.has('claims') ? (validateClaimGraph(json('claims')) ?? undefined) : undefined; + meta.manifestDigest = manifestDigest; meta.packDigest = storedPackDigest; + return { meta, lexicon: new Map(entries), postings, blocks, headings, docIds, namespaces, blockTokenLens, chunks: parsedChunks, semantic, claimGraph }; +} + +function u32Bytes(values: Uint32Array): Uint8Array { const out = new Uint8Array(values.length * 4); const dv = new DataView(out.buffer); for (let i=0;i length) throw new Error('Digest exceeds v4 header field.'); out.set(bytes, offset); } +function readAscii(out: Uint8Array, offset: number, length: number): string { return new TextDecoder().decode(out.slice(offset, offset + length)).replace(/\0+$/, ''); } + +function analyzeChunk(text: string, id: number, pack: Pack): PackChunk { + const lines = text.split(/\r?\n/); + const codeSymbols = [...text.matchAll(/\b(?:class|interface|function|const|let|var|def|fn)\s+([A-Za-z_$][\w$]*)/g)].map((m) => m[1]); + const paths = [...text.matchAll(/(?:^|[\s"'`])((?:\.\.?\/|[A-Za-z]:[\\/])[\w./\\-]+)/g)].map((m) => m[1]); + const tableLines = lines.filter((line) => /^\s*\|.*\|\s*$/.test(line)); + const tableHeaders = tableLines[0] ? tableLines[0].split('|').map((x) => x.trim()).filter(Boolean) : []; + const kind: PackChunk['kind'] = codeSymbols.length || /```/.test(text) ? 'code' : tableLines.length > 1 ? 'table' : /^\s*#/.test(text) ? 'markdown' : 'document'; + return { id, text, displayText: text, fieldedText: [pack.headings?.[id] ?? '', text, ...codeSymbols, ...paths].filter(Boolean).join(' '), heading: pack.headings?.[id] ?? null, docId: pack.docIds?.[id] ?? null, namespace: pack.namespaces?.[id] ?? null, len: pack.blockTokenLens?.[id] ?? 0, span: { start: 0, end: text.length, lineStart: 1, lineEnd: lines.length }, kind, codeSymbols: [...new Set(codeSymbols)].sort(), paths: [...new Set(paths)].sort(), tableHeaders }; +} diff --git a/packages/core/src/quality/diversify.ts b/packages/core/src/quality/diversify.ts index 384461b..d7ab3bc 100644 --- a/packages/core/src/quality/diversify.ts +++ b/packages/core/src/quality/diversify.ts @@ -15,7 +15,7 @@ export function diversifyAndDedupe( opts: DiversifyOptions ): HitLike[] { const { k, lambda = 0.8, simThreshold = 0.92, sim = (a, b) => jaccard5(a.text, b.text) } = opts; - const pool = [...hits].sort((a, b) => b.score - a.score); + const pool = [...hits].sort((a, b) => b.score - a.score || a.blockId - b.blockId); const kept: HitLike[] = []; while (pool.length && kept.length < k) { diff --git a/packages/core/src/query.ts b/packages/core/src/query.ts index f827864..aee119a 100644 --- a/packages/core/src/query.ts +++ b/packages/core/src/query.ts @@ -5,7 +5,7 @@ * - REQUIRED phrase enforcement (quoted and requirePhrases) * - Proximity bonus based on min cover span * - Optional heading overlap boost - * - KNS numeric-signature tie-breaker (tiny) + * - Stable block-id tie-breaking * - Near-duplicate suppression + MMR diversity */ @@ -14,12 +14,12 @@ import { rankBM25L } from "./rank.js"; import type { Pack } from "./pack.js"; import { minCoverSpan, proximityMultiplier } from "./quality/proximity.js"; import { diversifyAndDedupe } from "./quality/diversify.js"; -import { knsSignature, knsDistance } from "./quality/signature.js"; import { decodeScaleF16, quantizeEmbeddingInt8L2Norm } from "./semantic.js"; import { expandQueryWithGraph } from "./graph/query_expand.js"; import type { RetrievalEvidence, SemanticSidecar } from "./semantic/types.js"; import { rerankCandidates } from "./semantic/rerank.js"; import { parseSidecar } from "./semantic/sidecar.js"; +import { createRetrievalPlan, type RetrievalPlan } from './retrieval_plan.js'; export type QueryOptions = { topK?: number; @@ -62,6 +62,28 @@ export type QueryOptions = { }; }; +export type QueryWithPlanResult = { hits: Hit[]; plan: RetrievalPlan }; + +export function queryWithPlan(pack: Pack, q: string, opts: QueryOptions = {}): QueryWithPlanResult { + const namespace = normalizeNamespaceFilter(opts.namespace); + const source = normalizeSourceFilter(opts.source); + const requiredPhrases = [...parsePhrases(q).map((seq) => seq.map(normalize)), ...(opts.requirePhrases ?? []).map((phrase) => tokenize(phrase).map((token) => token.term))]; + const plan = createRetrievalPlan({ + version: 'retrieval-v4.0', + analyzer: pack.meta.analyzer ? { id: pack.meta.analyzer.id, digest: pack.meta.analyzer.digest } : undefined, + normalize: 'unicode-nfkd-diacritic-fold-v1', + scope: { namespace: [...namespace].sort(), source: [...source].sort(), requiredPhrases }, + generate: ['lexical-postings', ...(opts.queryExpansion?.enabled === false ? [] : ['pseudo-relevance-expansion']), ...(opts.graph?.expand ? ['claim-graph-expansion'] : []), ...(pack.chunks ? ['fielded-chunks'] : [])], + constrain: ['namespace', 'source', 'required-phrases'], + expand: { enabled: opts.queryExpansion?.enabled !== false, graph: opts.graph?.expand === true }, + rescore: ['bm25l', 'proximity', 'heading', ...(opts.semantic?.enabled ? ['semantic-grounded-rerank'] : [])], + semantic: { enabled: opts.semantic?.enabled === true, grounded: true }, + diversify: 'stable-id-mmr-v1', + }); + const hits = query(pack, q, { ...opts, __planHash: plan.planHash } as QueryOptions & { __planHash?: string }); + return { hits, plan }; +} + export function validateQueryOptions(opts?: QueryOptions): void { if (!opts) return; if (opts.topK !== undefined && (!Number.isInteger(opts.topK) || opts.topK < 1)) { @@ -161,9 +183,64 @@ export type Hit = { evidence?: RetrievalEvidence; }; +type Candidate = { + tf: Map; + pos: Map; + hasPhrase?: boolean; + headingScore?: number; + fieldScore?: number; +}; + +/** + * Apply every hard retrieval boundary to a candidate set. + * + * This function is deliberately shared by the initial lexical scan and every + * later candidate-producing stage. Relevance stages may add or reorder + * candidates, but they must never be able to widen a caller's scope. + */ +export function applyHardConstraints( + pack: Pack, + candidates: Map, + constraints: { + namespace?: Set; + source?: Set; + requiredPhrases?: string[][]; + }, +): void { + const namespaceFilter = constraints.namespace ?? new Set(); + const sourceFilter = constraints.source ?? new Set(); + const requiredPhrases = constraints.requiredPhrases ?? []; + + for (const [bid, data] of [...candidates]) { + if (namespaceFilter.size > 0) { + const namespace = typeof pack.namespaces?.[bid] === "string" ? normalize(pack.namespaces[bid] as string) : ""; + if (!namespace || !namespaceFilter.has(namespace)) { + candidates.delete(bid); + continue; + } + } + if (sourceFilter.size > 0) { + const source = typeof pack.docIds?.[bid] === "string" ? normalize(pack.docIds[bid] as string) : ""; + if (!source || !sourceFilter.has(source)) { + candidates.delete(bid); + continue; + } + } + if (requiredPhrases.length > 0) { + const text = pack.blocks[bid] ?? ""; + if (!requiredPhrases.every((phrase) => containsPhrase(text, phrase))) { + candidates.delete(bid); + continue; + } + data.hasPhrase = true; + } + } +} + export function query(pack: Pack, q: string, opts: QueryOptions = {}): Hit[] { validateQueryOptions(opts); const topK = opts.topK ?? 10; + const planHash = (opts as QueryOptions & { __planHash?: string }).__planHash; const minScore = Number.isFinite(opts.minScore) ? Math.max(0, opts.minScore as number) : 0; const expansionOpts = { enabled: opts.queryExpansion?.enabled ?? true, @@ -225,7 +302,7 @@ export function query(pack: Pack, q: string, opts: QueryOptions = {}): Hit[] { // --- Candidate map const candidates = new Map< number, - { tf: Map; pos: Map; hasPhrase?: boolean; headingScore?: number } + Candidate >(); // Query-time document frequency collection for BM25 IDF. @@ -281,6 +358,28 @@ export function query(pack: Pack, q: string, opts: QueryOptions = {}): Hit[] { scanForTermIds(new Map(Array.from(termSet.values(), (tid) => [tid, 1]))); } + // Fielded v4 chunks provide deterministic title/heading/code/path/table + // signals. They can add grounded lexical candidates, but are constrained + // immediately below like every other candidate-producing stage. + if (pack.chunks) { + for (const chunk of pack.chunks) { + const fieldTokens = tokenize(chunk.fieldedText ?? chunk.text).map((token) => token.term); + const matched = normTokens.filter((token) => fieldTokens.includes(token)); + if (matched.length === 0) continue; + const entry: Candidate = candidates.get(chunk.id) ?? { tf: new Map(), pos: new Map() }; + entry.fieldScore = new Set(matched).size / Math.max(1, new Set(normTokens).size); + for (const token of new Set(matched)) { + const tid = pack.lexicon.get(token); + if (tid === undefined) continue; + const positions = fieldTokens.flatMap((value, position) => value === token ? [position] : []); + if (!entry.tf.has(tid)) entry.tf.set(tid, Math.max(1, positions.length)); + if (!entry.pos.has(tid)) entry.pos.set(tid, positions); + } + candidates.set(chunk.id, entry); + } + applyHardConstraints(pack, candidates, { namespace: namespaceFilter, source: sourceFilter, requiredPhrases }); + } + // 2) Phrase-first rescue: // If nothing matched the free tokens, but we do have required phrases, // build a fallback term set from ALL tokens that appear in those phrases and scan again. @@ -297,37 +396,12 @@ export function query(pack: Pack, q: string, opts: QueryOptions = {}): Hit[] { } } - // --- Namespace filtering - if (namespaceFilter.size > 0) { - for (const bid of [...candidates.keys()]) { - const ns = pack.namespaces?.[bid]; - const normalizedNs = typeof ns === "string" ? normalize(ns) : ""; - if (!normalizedNs || !namespaceFilter.has(normalizedNs)) { - candidates.delete(bid); - } - } - } - - // --- Source/docId filtering - if (sourceFilter.size > 0) { - for (const bid of [...candidates.keys()]) { - const source = pack.docIds?.[bid]; - const normalizedSource = typeof source === "string" ? normalize(source) : ""; - if (!normalizedSource || !sourceFilter.has(normalizedSource)) { - candidates.delete(bid); - } - } - } - - // --- Phrase enforcement (now that we have some candidates) - if (requiredPhrases.length > 0) { - for (const [bid, data] of [...candidates]) { - const text = pack.blocks[bid] || ""; - const ok = requiredPhrases.every((seq) => containsPhrase(text, seq)); - if (!ok) candidates.delete(bid); - else data.hasPhrase = true; - } - } else if (quoted.length > 0) { + applyHardConstraints(pack, candidates, { + namespace: namespaceFilter, + source: sourceFilter, + requiredPhrases, + }); + if (requiredPhrases.length === 0 && quoted.length > 0) { for (const [bid, data] of candidates) { const text = pack.blocks[bid] || ""; data.hasPhrase = quoted.some((seq) => containsPhrase(text, seq)); @@ -366,6 +440,11 @@ export function query(pack: Pack, q: string, opts: QueryOptions = {}): Hit[] { const expansionWeights = deriveExpansionTerms(pack, prelim, termSet, requiredPhrases, expansionOpts); if (expansionWeights.size > 0) { scanForTermIds(expansionWeights, { collectPositions: false, createCandidates: true }); + applyHardConstraints(pack, candidates, { + namespace: namespaceFilter, + source: sourceFilter, + requiredPhrases, + }); prelim = rankBM25L(candidates, avgLen, docCount, dfs, pack.blockTokenLens, { proximityBonus: (cand) => proximityMultiplier(minCoverSpan(cand.pos)), }); @@ -392,14 +471,13 @@ export function query(pack: Pack, q: string, opts: QueryOptions = {}): Hit[] { const retrievalMode = semanticScores ? "hybrid" : "lexical"; - // --- KNS tie-breaker + de-dup/MMR - const qSig = knsSignature(normalize(q)); + // --- Stable block-id ordering + de-dup/MMR. Relevance scores are never + // modified by an unrelated query/text signature. const pool = prelim.slice(0, topK * 5).map((r) => { const text = pack.blocks[r.blockId] || ""; - const boost = 1 + 0.02 * (1 - knsDistance(qSig, knsSignature(text))); return { blockId: r.blockId, - score: r.score * boost, + score: r.score, text, source: pack.docIds?.[r.blockId] ?? undefined, namespace: pack.namespaces?.[r.blockId] ?? undefined, @@ -408,6 +486,12 @@ export function query(pack: Pack, q: string, opts: QueryOptions = {}): Hit[] { lexicalScore: originalLexicalScores.get(r.blockId) ?? r.score, semanticScore: semanticScores?.get(r.blockId), blendedScore: blendedScores?.get(r.blockId), + scoreBreakdown: { + lexical: originalLexicalScores.get(r.blockId) ?? r.score, + ...(semanticScores?.has(r.blockId) ? { semantic: semanticScores.get(r.blockId) } : {}), + ...(blendedScores?.has(r.blockId) ? { blended: blendedScores.get(r.blockId) } : {}), + }, + planHash, modelId: semanticOpts.provider?.modelId ?? semanticOpts.sidecar?.modelId, }, }; diff --git a/packages/core/src/rank.ts b/packages/core/src/rank.ts index 1fb05ae..c481542 100644 --- a/packages/core/src/rank.ts +++ b/packages/core/src/rank.ts @@ -13,11 +13,12 @@ export type RankOptions = { pos?: Map; hasPhrase?: boolean; headingScore?: number; + fieldScore?: number; }) => number; }; export function rankBM25L( - candidates: Map; pos?: Map; hasPhrase?: boolean; headingScore?: number }>, + candidates: Map; pos?: Map; hasPhrase?: boolean; headingScore?: number; fieldScore?: number }>, avgLen: number, docCount: number, dfs: Map, @@ -43,9 +44,10 @@ export function rankBM25L( if (opts.proximityBonus) score *= opts.proximityBonus(data) ?? 1; if (data.hasPhrase) score *= 1 + phraseBoost; if (data.headingScore) score *= 1 + headingBoost * data.headingScore; + if (data.fieldScore) score *= 1 + 0.2 * Math.min(1, data.fieldScore); results.push({ blockId: bid, score }); } - results.sort((a, b2) => b2.score - a.score); + results.sort((a, b2) => b2.score - a.score || a.blockId - b2.blockId); return results; } diff --git a/packages/core/src/receipt.ts b/packages/core/src/receipt.ts new file mode 100644 index 0000000..b02c6cc --- /dev/null +++ b/packages/core/src/receipt.ts @@ -0,0 +1,79 @@ +import type { Pack } from './pack.runtime.js'; +import type { QueryOptions, QueryWithPlanResult } from './query.js'; +import { queryWithPlan } from './query.js'; +import { createPackFingerprint } from './semantic/sidecar.js'; +import { getTextEncoder } from './utils/utf8.js'; +import { sha256Hex } from './utils/sha256.js'; +import { normalize } from './tokenize.js'; + +export type ReceiptDecision = 'answer' | 'clarify' | 'abstain'; +export type EvidenceSpan = { blockId: number; source?: string; start: number; end: number; text: string }; +export type QueryReceipt = { + version: 'receipt-v1'; + packDigest: string; + manifestDigest?: string; + engine: 'knolo-core'; + analyzer?: { id: string; digest: string }; + normalizedQuery: string; + plan: QueryWithPlanResult['plan']; + hits: Array<{ blockId: number; score: number; source?: string; namespace?: string; lexicalScore?: number; semanticScore?: number; blendedScore?: number; sourceDigest?: string; spans: EvidenceSpan[] }>; + answerability: number; + decision: ReceiptDecision; + reasons: string[]; + replayHash: string; + verified: boolean; +}; + +export type ReceiptOptions = QueryOptions & { policy?: { minAnswerability?: number; onEmpty?: ReceiptDecision } }; + +export function queryWithReceipt(pack: Pack, query: string, options: ReceiptOptions = {}): { hits: QueryWithPlanResult['hits']; receipt: QueryReceipt } { + const { policy, ...queryOptions } = options; + const result = queryWithPlan(pack, query, queryOptions); + const answerability = calculateAnswerability(result.hits); + const threshold = policy?.minAnswerability ?? 0.35; + const decision: ReceiptDecision = result.hits.length === 0 ? (policy?.onEmpty ?? 'abstain') : answerability >= threshold ? 'answer' : 'clarify'; + const reasons = result.hits.length === 0 ? ['no-grounded-candidates'] : decision === 'clarify' ? ['low-answerability'] : []; + const hits = result.hits.map((hit) => ({ + blockId: hit.blockId, + score: hit.score, + source: hit.source, + namespace: hit.namespace, + lexicalScore: hit.evidence?.lexicalScore, + semanticScore: hit.evidence?.semanticScore, + blendedScore: hit.evidence?.blendedScore, + sourceDigest: sourceDigest(pack, hit.blockId), + spans: [makeEvidenceSpan(pack, hit.blockId, query)], + })); + const partial = { version: 'receipt-v1' as const, packDigest: packDigest(pack), manifestDigest: pack.meta.manifestDigest, engine: 'knolo-core' as const, analyzer: result.plan.analyzer, normalizedQuery: normalize(query).replace(/\s+/g, ' ').trim(), plan: result.plan, hits, answerability, decision, reasons }; + const replayHash = `sha256-${sha256Hex(getTextEncoder().encode(JSON.stringify(partial)))}`; + return { hits: result.hits, receipt: { ...partial, replayHash, verified: true } }; +} + +export function verifyReceipt(receipt: QueryReceipt, pack: Pack): void { + if (!receipt || receipt.version !== 'receipt-v1') throw new Error('Unsupported receipt version.'); + if (receipt.packDigest !== packDigest(pack)) throw new Error(`Receipt pack digest mismatch: expected ${packDigest(pack)}, got ${receipt.packDigest}.`); + if (receipt.manifestDigest && receipt.manifestDigest !== pack.meta.manifestDigest) throw new Error('Receipt manifest digest mismatch.'); + for (const hit of receipt.hits) { + if (!Number.isInteger(hit.blockId) || hit.blockId < 0 || hit.blockId >= pack.blocks.length) throw new Error(`Receipt references invalid block ${hit.blockId}.`); + if (hit.sourceDigest !== sourceDigest(pack, hit.blockId)) throw new Error(`Receipt source digest mismatch for block ${hit.blockId}.`); + for (const span of hit.spans) { + if (span.start < 0 || span.end < span.start || span.end > pack.blocks[hit.blockId].length || span.text !== pack.blocks[hit.blockId].slice(span.start, span.end)) throw new Error(`Receipt evidence span mismatch for block ${hit.blockId}.`); + } + } + const { replayHash, verified, ...partial } = receipt; + const expected = `sha256-${sha256Hex(getTextEncoder().encode(JSON.stringify(partial)))}`; + if (replayHash !== expected) throw new Error('Receipt replay hash mismatch.'); + void verified; +} + +export function packDigest(pack: Pack): string { + return pack.meta.packDigest ?? `sha256-${sha256Hex(getTextEncoder().encode(JSON.stringify({ version: pack.meta.version, blocks: pack.blocks, docIds: pack.docIds ?? [], namespaces: pack.namespaces ?? [], analyzer: pack.meta.analyzer ?? null })))}`; +} + +function sourceDigest(pack: Pack, blockId: number): string { return `sha256-${sha256Hex(getTextEncoder().encode(`${pack.docIds?.[blockId] ?? ''}\n${pack.blocks[blockId] ?? ''}`))}`; } +function calculateAnswerability(hits: Array<{ score: number }>): number { if (!hits.length) return 0; const top = Math.max(0, hits[0].score); return Math.max(0, Math.min(1, top / (top + 1))); } +function makeEvidenceSpan(pack: Pack, blockId: number, query: string): EvidenceSpan { + const text = pack.blocks[blockId] ?? ''; const terms = normalize(query).split(/\s+/).filter(Boolean); const lower = text.toLocaleLowerCase(); const needle = terms[0] ?? ''; + const start = needle ? lower.indexOf(needle) : -1; const safeStart = start >= 0 ? start : 0; const end = start >= 0 ? Math.min(text.length, safeStart + needle.length) : text.length; + return { blockId, source: pack.docIds?.[blockId] ?? undefined, start: safeStart, end, text: text.slice(safeStart, end) }; +} diff --git a/packages/core/src/retrieval_plan.ts b/packages/core/src/retrieval_plan.ts new file mode 100644 index 0000000..847ae98 --- /dev/null +++ b/packages/core/src/retrieval_plan.ts @@ -0,0 +1,21 @@ +import { getTextEncoder } from './utils/utf8.js'; +import { sha256Hex } from './utils/sha256.js'; + +export type RetrievalPlan = { + version: 'retrieval-v4.0'; + analyzer?: { id: string; digest: string }; + normalize: string; + scope: { namespace?: string[]; source?: string[]; requiredPhrases?: string[][] }; + generate: string[]; + constrain: string[]; + expand: { enabled: boolean; graph: boolean }; + rescore: string[]; + semantic: { enabled: boolean; grounded: boolean }; + diversify: string; + planHash: string; +}; + +export function createRetrievalPlan(input: Omit): RetrievalPlan { + const canonical = JSON.stringify(input); + return { ...input, planHash: `sha256-${sha256Hex(getTextEncoder().encode(canonical))}` }; +} diff --git a/packages/core/src/semantic/types.ts b/packages/core/src/semantic/types.ts index 86e2653..e41789b 100644 --- a/packages/core/src/semantic/types.ts +++ b/packages/core/src/semantic/types.ts @@ -43,5 +43,15 @@ export type RetrievalEvidence = { lexicalScore?: number; semanticScore?: number; blendedScore?: number; + /** Stable diagnostics for replay/debugging; never used as hidden ranking input. */ + scoreBreakdown?: { + lexical?: number; + semantic?: number; + blended?: number; + phrase?: number; + proximity?: number; + heading?: number; + }; + planHash?: string; modelId?: string; }; diff --git a/packages/core/src/utils/sha256.ts b/packages/core/src/utils/sha256.ts new file mode 100644 index 0000000..e26d53e --- /dev/null +++ b/packages/core/src/utils/sha256.ts @@ -0,0 +1,50 @@ +// Small runtime-safe SHA-256 implementation used for pack identities. +// It intentionally avoids Node-only crypto so browser/RN readers can verify packs. +const K = new Uint32Array([ + 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5,0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5, + 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3,0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174, + 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc,0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da, + 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7,0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967, + 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13,0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85, + 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3,0xd192e819,0xd6990624,0xf40e3585,0x106aa070, + 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5,0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3, + 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208,0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2, +]); + +const rotr = (x: number, n: number) => (x >>> n) | (x << (32 - n)); + +export function sha256Hex(input: Uint8Array): string { + const bitLength = input.length * 8; + const paddedLength = (((input.length + 9 + 63) >> 6) << 6); + const data = new Uint8Array(paddedLength); + data.set(input); + data[input.length] = 0x80; + const dv = new DataView(data.buffer); + dv.setUint32(paddedLength - 4, bitLength >>> 0, false); + dv.setUint32(paddedLength - 8, Math.floor(bitLength / 0x100000000) >>> 0, false); + + let h0 = 0x6a09e667, h1 = 0xbb67ae85, h2 = 0x3c6ef372, h3 = 0xa54ff53a; + let h4 = 0x510e527f, h5 = 0x9b05688c, h6 = 0x1f83d9ab, h7 = 0x5be0cd19; + const w = new Uint32Array(64); + for (let off = 0; off < data.length; off += 64) { + for (let i = 0; i < 16; i++) w[i] = dv.getUint32(off + i * 4, false); + for (let i = 16; i < 64; i++) { + const s0 = rotr(w[i - 15], 7) ^ rotr(w[i - 15], 18) ^ (w[i - 15] >>> 3); + const s1 = rotr(w[i - 2], 17) ^ rotr(w[i - 2], 19) ^ (w[i - 2] >>> 10); + w[i] = (w[i - 16] + s0 + w[i - 7] + s1) >>> 0; + } + let a=h0,b=h1,c=h2,d=h3,e=h4,f=h5,g=h6,hh=h7; + for (let i = 0; i < 64; i++) { + const S1 = rotr(e,6)^rotr(e,11)^rotr(e,25); + const ch = (e & f) ^ (~e & g); + const t1 = (hh + S1 + ch + K[i] + w[i]) >>> 0; + const S0 = rotr(a,2)^rotr(a,13)^rotr(a,22); + const maj = (a & b) ^ (a & c) ^ (b & c); + const t2 = (S0 + maj) >>> 0; + hh=g; g=f; f=e; e=(d+t1)>>>0; d=c; c=b; b=a; a=(t1+t2)>>>0; + } + h0=(h0+a)>>>0; h1=(h1+b)>>>0; h2=(h2+c)>>>0; h3=(h3+d)>>>0; + h4=(h4+e)>>>0; h5=(h5+f)>>>0; h6=(h6+g)>>>0; h7=(h7+hh)>>>0; + } + return [h0,h1,h2,h3,h4,h5,h6,h7].map((x)=>x.toString(16).padStart(8,'0')).join(''); +} diff --git a/packages/create-knolo-app/README.md b/packages/create-knolo-app/README.md index ac33d55..7a6c497 100644 --- a/packages/create-knolo-app/README.md +++ b/packages/create-knolo-app/README.md @@ -1,5 +1,9 @@ # create-knolo-app +Generated applications target the Knolo v4 TypeScript runtime. Packs are v4 by +default and can be checked with `knolo verify`; Python/Rust/ICP adapters remain +legacy profiles until their TrustBench conformance is complete. + Bootstrap a new KnoLo-powered application in seconds. Creates a Next.js app preconfigured with: @@ -111,5 +115,4 @@ You can experiment directly in the playground UI. ## πŸ“„ License -MIT - +Apache-2.0 diff --git a/packages/create-knolo-app/template/README.md b/packages/create-knolo-app/template/README.md index 9be38c4..56bd399 100644 --- a/packages/create-knolo-app/template/README.md +++ b/packages/create-knolo-app/template/README.md @@ -12,5 +12,8 @@ Open http://localhost:3000 and ask questions about files in `./docs`. ## License -MIT +Apache-2.0 +# Knolo v4 application template +This generated app uses `@knolo/core` 4.x and v4 packs by default. Use the +CLI’s inspect/verify commands when distributing packs or receipts. diff --git a/packages/create-knolo-app/template/package.json b/packages/create-knolo-app/template/package.json index efdaeb1..dfdbd2e 100644 --- a/packages/create-knolo-app/template/package.json +++ b/packages/create-knolo-app/template/package.json @@ -11,13 +11,13 @@ "knolo:dev": "knolo dev" }, "dependencies": { - "@knolo/core": "^3.1.2", + "@knolo/core": "^4.0.0", "next": "^14.2.5", "react": "^18.3.1", "react-dom": "^18.3.1" }, "devDependencies": { - "@knolo/cli": "^3.1.2" + "@knolo/cli": "^4.0.0" }, "engines": { "node": ">=20" diff --git a/packages/icp-canister/CHANGELOG.md b/packages/icp-canister/CHANGELOG.md index 5566c2f..beaecb2 100644 --- a/packages/icp-canister/CHANGELOG.md +++ b/packages/icp-canister/CHANGELOG.md @@ -29,3 +29,10 @@ All notable changes to the ICP canister adapter will be documented in this file. ### Notes - Phase 2 now persists `.knolo` bytes and label across upgrades. - Write methods require the caller to be a canister controller (local `dfx` identity qualifies after deploy). +# Unreleased + +- Phase 2 compatibility note: the TypeScript builder emits v4 by default, while the CLI ICP build workflow explicitly emits legacy v3 packs until the canister reader gains a v4 capability profile. +- Phase 3 note: analyzer profiles and retrieval-plan hashes are TypeScript-core features; the canister remains on the legacy v3 capability profile until it gains v4 field/chunk support. +- Phase 4 note: receipt generation/verification is currently a TypeScript-core capability; ICP support should follow its v4 reader and resource-limit profile. +- Phase 5 note: shared conformance fixtures and TrustBench currently certify only the TypeScript runtime; ICP remains a legacy-v3 profile until v4 equivalence is implemented. +- Future ICP v4 work must implement the shared magic/header, section-directory bounds checks, and SHA-256 digest verification before changing that default. diff --git a/packages/semantic-ollama/package.json b/packages/semantic-ollama/package.json index 360cfd0..d45f662 100644 --- a/packages/semantic-ollama/package.json +++ b/packages/semantic-ollama/package.json @@ -11,7 +11,7 @@ "build": "tsc -p tsconfig.json" }, "dependencies": { - "@knolo/core": "^3.2.1" + "@knolo/core": "^4.0.0" }, "devDependencies": { "typescript": "^5.5.0" diff --git a/tests/icp-local/README.md b/tests/icp-local/README.md index 626b7ac..1281f99 100644 --- a/tests/icp-local/README.md +++ b/tests/icp-local/README.md @@ -1,5 +1,9 @@ # ICP local manual harness +This harness intentionally exercises the legacy ICP v3 capability profile. +The main TypeScript build emits v4 packs; use the ICP CLI builder, which opts +into v3 for the current canister reader. + Deploy the Knolo knowledge canister on a **local dfx replica**, seed it with **generated dummy data**, and exercise it from: - `knolo icp` CLI / `dfx canister call` diff --git a/trustbench/evaluator.mjs b/trustbench/evaluator.mjs new file mode 100644 index 0000000..4b00b0d --- /dev/null +++ b/trustbench/evaluator.mjs @@ -0,0 +1,37 @@ +import { readFile } from 'node:fs/promises'; +import { mountPack, queryWithReceipt, verifyReceipt } from '../packages/core/dist/index.js'; + +export async function evaluatePack(packBytes, queries) { + const pack = await mountPack({ src: packBytes }); + const rows = []; + for (const spec of queries) { + const result = queryWithReceipt(pack, spec.query, spec.options ?? {}); + verifyReceipt(result.receipt, pack); + const hits = result.hits.map((hit) => ({ id: hit.source ?? `block:${hit.blockId}`, blockId: hit.blockId, score: Number(hit.score.toFixed(6)) })); + rows.push({ id: spec.id, query: spec.query, hits, decision: result.receipt.decision, planHash: result.receipt.plan.planHash, receiptVerified: true, metrics: retrievalMetrics(hits, spec.relevant ?? []) }); + } + return { contract: 'retrieval-v4.0', packVersion: pack.meta.version, packFormat: pack.meta.format ?? `v${pack.meta.version}`, aggregate: aggregateMetrics(rows), rows }; +} + +export async function loadQueries(path) { + const raw = await readFile(path, 'utf8'); + return raw.split(/\r?\n/).filter(Boolean).map((line) => JSON.parse(line)); +} + +function retrievalMetrics(hits, relevant) { + const wanted = new Set(relevant); const ranked = hits.map((hit) => hit.id); + const relevantHits = ranked.filter((id) => wanted.has(id)); + const recallAtK = wanted.size ? relevantHits.length / wanted.size : (ranked.length ? 0 : 1); + const first = ranked.findIndex((id) => wanted.has(id)); + const dcg = ranked.reduce((sum, id, index) => sum + (wanted.has(id) ? 1 / Math.log2(index + 2) : 0), 0); + const ideal = [...wanted].slice(0, ranked.length).reduce((sum, _id, index) => sum + 1 / Math.log2(index + 2), 0); + return { recallAtK: Number(recallAtK.toFixed(6)), reciprocalRank: first < 0 ? 0 : Number((1 / (first + 1)).toFixed(6)), ndcgAtK: Number((ideal ? dcg / ideal : ranked.length ? 0 : 1).toFixed(6)), hitCount: ranked.length, relevantCount: relevantHits.length }; +} + +function aggregateMetrics(rows) { + const total = rows.length || 1; + const mean = (field) => Number((rows.reduce((sum, row) => sum + row.metrics[field], 0) / total).toFixed(6)); + const abstentionRows = rows.filter((row) => row.decision !== 'answer'); + const correctAbstentions = abstentionRows.filter((row) => row.metrics.relevantCount === 0).length; + return { recallAtK: mean('recallAtK'), mrrAtK: mean('reciprocalRank'), ndcgAtK: mean('ndcgAtK'), meanHitCount: mean('hitCount'), answerCount: rows.filter((row) => row.decision === 'answer').length, abstentionCount: abstentionRows.length, abstentionPrecision: Number((abstentionRows.length ? correctAbstentions / abstentionRows.length : 1).toFixed(6)), receiptsVerified: rows.every((row) => row.receiptVerified) }; +} diff --git a/trustbench/generate-fixtures.mjs b/trustbench/generate-fixtures.mjs new file mode 100644 index 0000000..79ea685 --- /dev/null +++ b/trustbench/generate-fixtures.mjs @@ -0,0 +1,34 @@ +import { mkdir, writeFile } from 'node:fs/promises'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { buildPack, mountPack } from '../packages/core/dist/index.js'; + +const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); +const packs = path.join(root, 'conformance/packs'); +await mkdir(path.join(packs, 'corrupted'), { recursive: true }); +const docs = [ + { id: 'intro.md', namespace: 'docs.alpha', heading: 'Alpha Intro', text: 'alpha beta' }, + { id: 'runtime.md', namespace: 'docs.beta', heading: 'Beta Runtime', text: 'beta gamma runtime' }, + { id: 'other.md', namespace: 'docs.alpha', heading: 'Alpha Reference', text: 'alpha beta' }, +]; +const v3 = await buildPack(docs, { format: 3 }); +const v4 = await buildPack(docs); +await writeFile(path.join(packs, 'standard-v3.knolo'), v3); +await writeFile(path.join(packs, 'verified-v4.knolo'), v4); +await writeFile(path.join(packs, 'minimal-v1.knolo'), await buildLegacyV1()); +await writeFile(path.join(packs, 'corrupted/truncated.knolo'), v4.slice(0, Math.max(1, v4.length - 7))); +const digestCorrupt = Uint8Array.from(v4); +digestCorrupt[digestCorrupt.length - 1] ^= 0xff; +await writeFile(path.join(packs, 'corrupted/section-digest.knolo'), digestCorrupt); + +async function buildLegacyV1() { + const enc = new TextEncoder(); + const meta = enc.encode(JSON.stringify({ version: 1, stats: { docs: 1, blocks: 1, terms: 0 } })); + const lex = enc.encode('[]'); const blocks = enc.encode(JSON.stringify(['legacy fixture'])); + const out = new Uint8Array(4 + meta.length + 4 + lex.length + 4 + 4 + blocks.length); + const dv = new DataView(out.buffer); let at = 0; + dv.setUint32(at, meta.length, true); at += 4; out.set(meta, at); at += meta.length; + dv.setUint32(at, lex.length, true); at += 4; out.set(lex, at); at += lex.length; + dv.setUint32(at, 0, true); at += 4; dv.setUint32(at, blocks.length, true); at += 4; out.set(blocks, at); + return out; +} diff --git a/trustbench/package.json b/trustbench/package.json new file mode 100644 index 0000000..67f5978 --- /dev/null +++ b/trustbench/package.json @@ -0,0 +1,10 @@ +{ + "name": "@knolo/trustbench", + "private": true, + "type": "module", + "scripts": { + "generate": "node generate-fixtures.mjs", + "run": "node run.mjs", + "test": "node test.mjs" + } +} diff --git a/trustbench/run.mjs b/trustbench/run.mjs new file mode 100644 index 0000000..baf6a12 --- /dev/null +++ b/trustbench/run.mjs @@ -0,0 +1,13 @@ +import { mkdir, readFile, writeFile } from 'node:fs/promises'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { evaluatePack, loadQueries } from './evaluator.mjs'; + +const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); +const packPath = path.join(root, 'conformance/packs/verified-v4.knolo'); +const queries = await loadQueries(path.join(root, 'conformance/queries/retrieval.jsonl')); +const output = await evaluatePack(await readFile(packPath), queries); +output.runtime = { name: 'typescript', version: process.version }; +await mkdir(path.join(root, 'conformance/expected'), { recursive: true }); +await writeFile(path.join(root, 'conformance/expected/retrieval-v4.0.json'), `${JSON.stringify(output, null, 2)}\n`); +console.log(JSON.stringify(output, null, 2)); diff --git a/trustbench/test.mjs b/trustbench/test.mjs new file mode 100644 index 0000000..2384c49 --- /dev/null +++ b/trustbench/test.mjs @@ -0,0 +1,17 @@ +import assert from 'node:assert/strict'; +import { readFile } from 'node:fs/promises'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { mountPack } from '../packages/core/dist/index.js'; +import { evaluatePack, loadQueries } from './evaluator.mjs'; + +const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); +const expected = JSON.parse(await readFile(path.join(root, 'conformance/expected/retrieval-v4.0.json'), 'utf8')); +const actual = await evaluatePack(await readFile(path.join(root, 'conformance/packs/verified-v4.knolo')), await loadQueries(path.join(root, 'conformance/queries/retrieval.jsonl'))); +const expectedContract = { ...expected }; delete expectedContract.runtime; +assert.deepEqual(actual, expectedContract, 'conformance output changed; regenerate expected output intentionally'); + +for (const name of ['truncated.knolo', 'section-digest.knolo']) { + await assert.rejects(async () => mountPack({ src: await readFile(path.join(root, 'conformance/packs/corrupted', name)) }), /invalid|mismatch|truncated/i, `${name} must fail closed`); +} +console.log('TrustBench conformance passed.'); From 9a6865797d260f2e5500db8cb6904233396b6155 Mon Sep 17 00:00:00 2001 From: Sam Paniagua Date: Wed, 12 Aug 2026 12:35:47 +0200 Subject: [PATCH 2/4] test: harden Phase 5 TrustBench review gate --- packages/core/CHANGELOG.md | 1 + trustbench/README.md | 25 +++++++++++++++++++++++++ trustbench/test.mjs | 11 +++++++++++ 3 files changed, 37 insertions(+) create mode 100644 trustbench/README.md diff --git a/packages/core/CHANGELOG.md b/packages/core/CHANGELOG.md index 3b4c8f4..b6e3f21 100644 --- a/packages/core/CHANGELOG.md +++ b/packages/core/CHANGELOG.md @@ -7,6 +7,7 @@ All notable changes to this project will be documented in this file. ### v4.0.0 / Phase 5 - Completed the TypeScript TrustBench reference profile and repository conformance documentation. - Added aggregate Recall@K, MRR, nDCG, hit-count, abstention-precision, and receipt-verification reporting. +- Added explicit v1/v3/v4 profile mounting checks and a TrustBench operator README. - Bumped `@knolo/core` to `4.0.0`; legacy v1–v3 readers remain supported. ### Changed diff --git a/trustbench/README.md b/trustbench/README.md new file mode 100644 index 0000000..1067207 --- /dev/null +++ b/trustbench/README.md @@ -0,0 +1,25 @@ +# Knolo TrustBench + +TrustBench is the reproducible Phase 5 quality and conformance harness for the +Knolo v4 TypeScript reference runtime. + +Run from the repository root: + +```bash +npm run trustbench:generate +npm run trustbench:test +``` + +`generate` rebuilds the checked-in v1/v3/v4 and corrupted fixtures, then writes +the canonical `conformance/expected/retrieval-v4.0.json` report. `test` checks +that report, verifies every committed pack profile mounts as expected, and +ensures corrupted v4 artifacts fail closed. + +The report includes Recall@K, MRR, nDCG, mean hit count, answer/abstention +counts, abstention precision, receipt verification, hit IDs, rounded scores, +decisions, and retrieval-plan hashes. Scores are rounded to six decimals; +identities, decisions, plan hashes, and corruption outcomes are exact. + +The TypeScript runtime is the reference profile. A Rust, Python, WASM, or ICP +profile should only be called equivalent after it consumes the same fixtures +and emits the same canonical result fields. diff --git a/trustbench/test.mjs b/trustbench/test.mjs index 2384c49..6b37ca6 100644 --- a/trustbench/test.mjs +++ b/trustbench/test.mjs @@ -11,6 +11,17 @@ const actual = await evaluatePack(await readFile(path.join(root, 'conformance/pa const expectedContract = { ...expected }; delete expectedContract.runtime; assert.deepEqual(actual, expectedContract, 'conformance output changed; regenerate expected output intentionally'); +const profileCases = [ + ['minimal-v1.knolo', 1], + ['standard-v3.knolo', 3], + ['verified-v4.knolo', 4], +]; +for (const [name, version] of profileCases) { + const pack = await mountPack({ src: await readFile(path.join(root, 'conformance/packs', name)) }); + assert.equal(pack.meta.version, version, `${name} must mount as v${version}`); + assert.ok(pack.blocks.length > 0, `${name} must contain at least one block`); +} + for (const name of ['truncated.knolo', 'section-digest.knolo']) { await assert.rejects(async () => mountPack({ src: await readFile(path.join(root, 'conformance/packs/corrupted', name)) }), /invalid|mismatch|truncated/i, `${name} must fail closed`); } From eb82a0746c7e8a9bcd2d9b214ed58f96913370e0 Mon Sep 17 00:00:00 2001 From: Sam Paniagua Date: Fri, 14 Aug 2026 15:13:29 +0200 Subject: [PATCH 3/4] fix: bind retrieval identity to query and pack state --- packages/core/src/query.ts | 43 +++++++++++++++++++++++++++++ packages/core/src/rank.ts | 4 +++ packages/core/src/receipt.ts | 21 +++++++++++++- packages/core/src/retrieval_plan.ts | 2 ++ 4 files changed, 69 insertions(+), 1 deletion(-) diff --git a/packages/core/src/query.ts b/packages/core/src/query.ts index aee119a..4326739 100644 --- a/packages/core/src/query.ts +++ b/packages/core/src/query.ts @@ -65,6 +65,7 @@ export type QueryOptions = { export type QueryWithPlanResult = { hits: Hit[]; plan: RetrievalPlan }; export function queryWithPlan(pack: Pack, q: string, opts: QueryOptions = {}): QueryWithPlanResult { + validateQueryOptions(opts); const namespace = normalizeNamespaceFilter(opts.namespace); const source = normalizeSourceFilter(opts.source); const requiredPhrases = [...parsePhrases(q).map((seq) => seq.map(normalize)), ...(opts.requirePhrases ?? []).map((phrase) => tokenize(phrase).map((token) => token.term))]; @@ -79,11 +80,53 @@ export function queryWithPlan(pack: Pack, q: string, opts: QueryOptions = {}): Q rescore: ['bm25l', 'proximity', 'heading', ...(opts.semantic?.enabled ? ['semantic-grounded-rerank'] : [])], semantic: { enabled: opts.semantic?.enabled === true, grounded: true }, diversify: 'stable-id-mmr-v1', + options: normalizePlanOptions(opts, namespace, source, requiredPhrases), }); const hits = query(pack, q, { ...opts, __planHash: plan.planHash } as QueryOptions & { __planHash?: string }); return { hits, plan }; } +function normalizePlanOptions(opts: QueryOptions, namespace: Iterable, source: Iterable, requiredPhrases: string[][]): Record { + return { + topK: opts.topK ?? 10, + minScore: opts.minScore ?? 0, + namespace: [...namespace].sort(), + source: [...source].sort(), + requiredPhrases, + queryExpansion: { + enabled: opts.queryExpansion?.enabled ?? true, + docs: opts.queryExpansion?.docs ?? 3, + terms: opts.queryExpansion?.terms ?? 4, + weight: opts.queryExpansion?.weight ?? 0.35, + minTermLength: opts.queryExpansion?.minTermLength ?? 3, + }, + graph: { + expand: opts.graph?.expand ?? false, + maxExtraTerms: opts.graph?.maxExtraTerms ?? 12, + predicates: opts.graph?.predicates + ? opts.graph.predicates.map((p) => normalize(p)).sort() + : ['defined_as', 'is', 'mentions', 'ref'], + }, + semantic: { + enabled: opts.semantic?.enabled ?? false, + mode: opts.semantic?.mode ?? 'rerank', + topN: opts.semantic?.topN ?? 50, + minLexConfidence: clamp01(opts.semantic?.minLexConfidence ?? 0.35), + minSemanticScore: opts.semantic?.minSemanticScore ?? null, + force: opts.semantic?.force ?? false, + blend: { + enabled: opts.semantic?.blend?.enabled ?? true, + wLex: opts.semantic?.blend?.wLex ?? 0.75, + wSem: opts.semantic?.blend?.wSem ?? 0.25, + }, + provider: opts.semantic?.provider ?? null, + sidecarPath: opts.semantic?.sidecarPath ?? null, + sidecar: opts.semantic?.sidecar ?? null, + queryEmbedding: opts.semantic?.queryEmbedding ? Array.from(opts.semantic.queryEmbedding) : null, + }, + }; +} + export function validateQueryOptions(opts?: QueryOptions): void { if (!opts) return; if (opts.topK !== undefined && (!Number.isInteger(opts.topK) || opts.topK < 1)) { diff --git a/packages/core/src/rank.ts b/packages/core/src/rank.ts index c481542..3550756 100644 --- a/packages/core/src/rank.ts +++ b/packages/core/src/rank.ts @@ -45,6 +45,10 @@ export function rankBM25L( if (data.hasPhrase) score *= 1 + phraseBoost; if (data.headingScore) score *= 1 + headingBoost * data.headingScore; if (data.fieldScore) score *= 1 + 0.2 * Math.min(1, data.fieldScore); + // A v4 field can contain terms that are intentionally absent from the + // body lexicon. Keep those grounded matches above zero so they remain + // meaningful candidates instead of becoming score-zero hits. + if (data.fieldScore) score += 0.2 * Math.min(1, data.fieldScore); results.push({ blockId: bid, score }); } diff --git a/packages/core/src/receipt.ts b/packages/core/src/receipt.ts index b02c6cc..dec656d 100644 --- a/packages/core/src/receipt.ts +++ b/packages/core/src/receipt.ts @@ -67,7 +67,26 @@ export function verifyReceipt(receipt: QueryReceipt, pack: Pack): void { } export function packDigest(pack: Pack): string { - return pack.meta.packDigest ?? `sha256-${sha256Hex(getTextEncoder().encode(JSON.stringify({ version: pack.meta.version, blocks: pack.blocks, docIds: pack.docIds ?? [], namespaces: pack.namespaces ?? [], analyzer: pack.meta.analyzer ?? null })))}`; + if (pack.meta.packDigest) return pack.meta.packDigest; + // Legacy packs have no stored whole-pack digest. Include every mounted + // retrieval input so receipts cannot be replayed against a behaviorally + // different pack with the same block text and ids. + const retrievalState = { + meta: { ...pack.meta, packDigest: undefined }, + blocks: pack.blocks, + headings: pack.headings ?? [], + docIds: pack.docIds ?? [], + namespaces: pack.namespaces ?? [], + blockTokenLens: pack.blockTokenLens ?? [], + chunks: pack.chunks ?? [], + lexicon: Array.from(pack.lexicon.entries()), + postings: Array.from(pack.postings), + semantic: pack.semantic + ? { ...pack.semantic, vecs: Array.from(pack.semantic.vecs), scales: pack.semantic.scales ? Array.from(pack.semantic.scales) : undefined } + : null, + claimGraph: pack.claimGraph ?? null, + }; + return `sha256-${sha256Hex(getTextEncoder().encode(JSON.stringify(retrievalState)))}`; } function sourceDigest(pack: Pack, blockId: number): string { return `sha256-${sha256Hex(getTextEncoder().encode(`${pack.docIds?.[blockId] ?? ''}\n${pack.blocks[blockId] ?? ''}`))}`; } diff --git a/packages/core/src/retrieval_plan.ts b/packages/core/src/retrieval_plan.ts index 847ae98..8a5d1f2 100644 --- a/packages/core/src/retrieval_plan.ts +++ b/packages/core/src/retrieval_plan.ts @@ -12,6 +12,8 @@ export type RetrievalPlan = { rescore: string[]; semantic: { enabled: boolean; grounded: boolean }; diversify: string; + /** Normalized result-affecting query settings included in planHash. */ + options: Record; planHash: string; }; From 5c49f374b5cbd068a7a9bf03ee073a7daa9d382e Mon Sep 17 00:00:00 2001 From: Sam Paniagua Date: Fri, 14 Aug 2026 17:21:36 +0200 Subject: [PATCH 4/4] fix: bind receipts to mounted retrieval state --- conformance/expected/retrieval-v4.0.json | 20 ++++++++++---------- packages/core/scripts/test.mjs | 13 +++++++++++++ packages/core/src/receipt.ts | 7 +++---- 3 files changed, 26 insertions(+), 14 deletions(-) diff --git a/conformance/expected/retrieval-v4.0.json b/conformance/expected/retrieval-v4.0.json index a8bcb90..b0107e9 100644 --- a/conformance/expected/retrieval-v4.0.json +++ b/conformance/expected/retrieval-v4.0.json @@ -20,16 +20,16 @@ { "id": "intro.md", "blockId": 0, - "score": 0.956856 + "score": 1.156856 }, { "id": "runtime.md", "blockId": 1, - "score": 0.797493 + "score": 0.897493 } ], "decision": "answer", - "planHash": "sha256-49e6fa300d07005e2bd75c0856d1c182fb9fd3cf36ca5b29308d0e1012ee18eb", + "planHash": "sha256-a37a075b3b1fb36ed90154c679bdb78b47de9065fcebe302223206ecbfc617b2", "receiptVerified": true, "metrics": { "recallAtK": 0.5, @@ -46,11 +46,11 @@ { "id": "intro.md", "blockId": 0, - "score": 1.069768 + "score": 1.269768 } ], "decision": "answer", - "planHash": "sha256-101807cf8e82cd778cf944f7cbb144943645d13468a196a3c0a60075ec460c24", + "planHash": "sha256-32f75441d37cffd6fd68d9bc6a76d67913163d121df50ab479d9218bc9ef64d5", "receiptVerified": true, "metrics": { "recallAtK": 0.5, @@ -67,11 +67,11 @@ { "id": "runtime.md", "blockId": 1, - "score": 2.36473 + "score": 2.56473 } ], "decision": "answer", - "planHash": "sha256-2ec416027feeeb28bcdca0d3a04ffc4c6525edce1b9dc6db70a95ce67890b016", + "planHash": "sha256-ff9b5b0daef232196dc01a57850d3f7d3bd8c953c5cece450f87c64d81875623", "receiptVerified": true, "metrics": { "recallAtK": 1, @@ -88,11 +88,11 @@ { "id": "intro.md", "blockId": 0, - "score": 1.441785 + "score": 1.641785 } ], "decision": "answer", - "planHash": "sha256-32734c7016b6298c1967bf54fd04f053f37cf01b5df1dcd3efa47e48d26dc31b", + "planHash": "sha256-934edee9ecea91cdf3927e5654f0f9195548e906e1d806e4fe9cb213162dde3b", "receiptVerified": true, "metrics": { "recallAtK": 0.5, @@ -107,7 +107,7 @@ "query": "alpha", "hits": [], "decision": "abstain", - "planHash": "sha256-fb84e31df21e151c53999de7ac6a8323b3f47630dc589ff94ccc02082f8883f8", + "planHash": "sha256-2d5f95949ddcbbe4bc76c14b16019e4dd2a58234ac78df8d99986cc859a38cbd", "receiptVerified": true, "metrics": { "recallAtK": 1, diff --git a/packages/core/scripts/test.mjs b/packages/core/scripts/test.mjs index 4af7d85..b623f9d 100644 --- a/packages/core/scripts/test.mjs +++ b/packages/core/scripts/test.mjs @@ -411,6 +411,18 @@ async function testPhase4ReceiptVerification() { assert.throws(() => verifyReceipt(tampered, pack), /evidence span mismatch|replay hash mismatch/); } +async function testRetrievalIdentityBindsOptionsAndPackState() { + const pack = await mountPack({ src: await buildPack([{ id: 'identity-doc', text: 'Identity binding protects retrieval replay.' }]) }); + const defaultPlan = queryWithPlan(pack, 'identity binding', { topK: 2 }).plan; + const constrainedPlan = queryWithPlan(pack, 'identity binding', { topK: 3, minScore: 0.25 }).plan; + assert.notEqual(defaultPlan.planHash, constrainedPlan.planHash, 'result-affecting query options must change the plan hash'); + + const receipt = queryWithReceipt(pack, 'identity binding', { topK: 2 }).receipt; + const alteredPack = { ...pack, postings: new Uint32Array(pack.postings) }; + alteredPack.postings[0] = alteredPack.postings[0] + 1; + assert.throws(() => verifyReceipt(receipt, alteredPack), /Receipt pack digest mismatch/); +} + async function testMinScoreFiltering() { const docs = [ { @@ -1873,6 +1885,7 @@ await testHardConstraintsSurviveQueryExpansion(); await testSourceTextIsPreserved(); await testPhase3AnalyzerAndRetrievalPlan(); await testPhase4ReceiptVerification(); +await testRetrievalIdentityBindsOptionsAndPackState(); await testSourceFiltering(); await testMinScoreFiltering(); await testContextPatchSourcePropagation(); diff --git a/packages/core/src/receipt.ts b/packages/core/src/receipt.ts index dec656d..a593c73 100644 --- a/packages/core/src/receipt.ts +++ b/packages/core/src/receipt.ts @@ -67,10 +67,9 @@ export function verifyReceipt(receipt: QueryReceipt, pack: Pack): void { } export function packDigest(pack: Pack): string { - if (pack.meta.packDigest) return pack.meta.packDigest; - // Legacy packs have no stored whole-pack digest. Include every mounted - // retrieval input so receipts cannot be replayed against a behaviorally - // different pack with the same block text and ids. + // Use the mounted retrieval state rather than trusting the container's + // stored digest. Packs are mutable in memory, and receipts must not remain + // valid after postings, chunks, metadata, or other retrieval inputs change. const retrievalState = { meta: { ...pack.meta, packDigest: undefined }, blocks: pack.blocks,