From 7f0e15d4c5077ce38d587230f5e5dc4b055b827c Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Wed, 28 Oct 2026 08:54:12 -0600 Subject: [PATCH 1/4] Skip reload markers in the audit history readers; return an empty history for an unaudited record getHistory and getHistoryOfRecord excluded eviction and lock-control entries but not the whole-table reload marker, so read_audit_log returned type:'reload' entries with a null id and, on RocksDB, grouped one into a user transaction. isNonHistoryType now owns the exclusion for both readers. The startTime replay and previousCount backfill keep handling the marker themselves (refuse or re-snapshot). getHistoryOfRecord threw for an entry with no audit position (an LMDB entry written without a local audit time), which read_audit_log by hash_value turned into HTTP 500. It now returns an empty history, as it does for an absent record. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_016q5qJ8dVxeE9co9xaEPQtp Dispatch-Task: pr-maint-9c4f1f0aa5c37bd0cdb8f905b8b9ebea --- resources/DESIGN.md | 2 + resources/Table.ts | 9 +- resources/auditStore.ts | 5 + .../resources/historyNonRecordEntries.test.js | 140 ++++++++++++++++++ 4 files changed, 151 insertions(+), 5 deletions(-) create mode 100644 unitTests/resources/historyNonRecordEntries.test.js diff --git a/resources/DESIGN.md b/resources/DESIGN.md index b3df22bdb1..944ccdcd1a 100644 --- a/resources/DESIGN.md +++ b/resources/DESIGN.md @@ -76,6 +76,8 @@ The synthesized target keeps the collection flag, or the batch is one null-keyed `getRecordAtTime` (`crdt.ts`) undoes only patches. Past a newer put, delete or invalidate it rebuilds forward (`reconstructForward`): carrying an image back returns a future state. A reversed field survives only when the reversal is exact. An unlinked put ends the chain without implying an empty past (`crdt.test.js`). +History readers skip entries that are not a record's history, through `isNonHistoryType` (`auditStore.ts`): `getHistory` and `getHistoryOfRecord` omit eviction, lock-control and the whole-table `reload` marker (`recordId: null`), so a marker never reaches `read_audit_log`. The `startTime` replay and the `previousCount` backfill must not use it: they handle a `reload` themselves (refuse or re-snapshot). A record with no audit position (an LMDB entry without `localTime`) has an empty history, like an absent record; carrying history across an audit-less base copy is harper#2212's design (`unitTests/resources/historyNonRecordEntries.test.js`). + ## Path routing & parameterised routes Parameterised routes sit outside the base map (`paramRoutes`), so exact and prefix matching stays fast and static routes win (`Resources.getMatch`). Literal > `:param` > `*`; a wildcard is last. `openApi.ts` and `components/mcp/resources.ts` each convert templates, so a syntax change touches both (`paramRoutes.test.js`). diff --git a/resources/Table.ts b/resources/Table.ts index ec7e6def2a..14fe621a52 100644 --- a/resources/Table.ts +++ b/resources/Table.ts @@ -223,6 +223,7 @@ import { raiseAuditFloor, boundedAuditPruneEnd, isLockControlType, + isNonHistoryType, isAuditEntryWrite, getAuditResumeFloor, isResumablePosition, @@ -11151,8 +11152,7 @@ export function makeTable(options): TableResourceClass { end: endTime, })) { await rest(); // yield to other async operations - if (auditRecord.tableId !== tableId || auditRecord.type === 'evict' || isLockControlType(auditRecord.type)) - continue; + if (auditRecord.tableId !== tableId || isNonHistoryType(auditRecord.type)) continue; let value; try { value = auditRecord.getValue(primaryStore, true, auditRecord.txnLogKey); @@ -11182,7 +11182,7 @@ export function makeTable(options): TableResourceClass { let nextVersion = isRocksDB ? resolveAuditHead(id, entry.version, entry.nodeId, entry.additionalAuditRefs).txnLogKey : entry.localTime; - if (!nextVersion) throw new Error('The entry does not have a local audit time'); + if (!nextVersion) return history; const count = 0; const auditWindow = 100; let loggedEntryError = false; @@ -11194,8 +11194,7 @@ export function makeTable(options): TableResourceClass { for (const auditRecord of auditStore.getRange({ start, end: nextVersion + 0.001 })) { if ( auditRecord.tableId === tableId && - auditRecord.type !== 'evict' && - !isLockControlType(auditRecord.type) && + !isNonHistoryType(auditRecord.type) && compareKeys(auditRecord.recordId, id) === 0 ) { let value; diff --git a/resources/auditStore.ts b/resources/auditStore.ts index 35d3a526ad..fce4a4d092 100644 --- a/resources/auditStore.ts +++ b/resources/auditStore.ts @@ -1382,6 +1382,11 @@ function isDecodableAction(action: number) { export function isLockControlType(type: unknown): boolean { return type === 'lockRelease' || type === 'lockBarrier'; } + +/** Entry types that belong to no record's history: eviction, lock control and the whole-table `reload` marker. */ +export function isNonHistoryType(type: unknown): boolean { + return type === 'evict' || type === 'reload' || isLockControlType(type); +} const ORIGINATING_OPERATIONS = { insert: 1, update: 2, diff --git a/unitTests/resources/historyNonRecordEntries.test.js b/unitTests/resources/historyNonRecordEntries.test.js new file mode 100644 index 0000000000..1ddb73dd35 --- /dev/null +++ b/unitTests/resources/historyNonRecordEntries.test.js @@ -0,0 +1,140 @@ +const assert = require('assert'); +const { setupTestDBPath } = require('../testUtils'); +const { table } = require('#src/resources/databases'); +const { transaction } = require('#src/resources/transaction'); +const { setMainIsWorker } = require('#js/server/threads/manageThreads'); +const { ResourceBridge } = require('#src/dataLayer/harperBridge/ResourceBridge'); +require('#src/server/serverHelpers/serverUtilities'); + +const isLMDB = process.env.HARPER_STORAGE_ENGINE === 'lmdb'; +const schema = 'history_non_record_entries'; + +describe('history readers and non-record audit entries', function () { + this.timeout(60_000); + let tableIndex = 0; + const bridge = new ResourceBridge(); + + before(function () { + setupTestDBPath(); + setMainIsWorker(true); + }); + + function newTable(database = schema) { + return table({ + database, + table: `HistoryNonRecord${++tableIndex}`, + attributes: [{ name: 'id', isPrimaryKey: true }, { name: 'name' }], + }); + } + + function auditEntries(T) { + const entries = []; + for (const entry of T.auditStore.getRange({ start: 1 })) { + if (entry.tableId === T.tableId) entries.push({ type: entry.type, txnLogKey: entry.txnLogKey }); + } + return entries; + } + + describe('reload marker', () => { + it('getHistory and getHistoryOfRecord omit it', async () => { + const T = newTable(); + await T.put({ id: 'a', name: 'one' }); + await T.writeReloadMarker(); + await T.put({ id: 'a', name: 'two' }); + assert.ok( + auditEntries(T).some((entry) => entry.type === 'reload'), + 'the marker is in the audit log' + ); + + const history = await Array.fromAsync(T.getHistory()); + assert.deepStrictEqual( + history.map((entry) => [entry.type, entry.id]), + [ + ['put', 'a'], + ['put', 'a'], + ] + ); + const ofRecord = await T.getHistoryOfRecord('a'); + assert.deepStrictEqual( + ofRecord.map((entry) => entry.value.name), + ['one', 'two'] + ); + }); + + it('a table with only a marker has no history', async () => { + const T = newTable(); + await T.writeReloadMarker(); + assert.deepStrictEqual( + auditEntries(T).map((entry) => entry.type), + ['reload'] + ); + assert.deepStrictEqual(await Array.fromAsync(T.getHistory()), []); + const groups = await Array.fromAsync( + await bridge.readAuditLog({ schema, table: T.tableName, search_type: 'timestamp', search_values: [0] }) + ); + assert.deepStrictEqual(groups, []); + }); + + it('read_audit_log reports only the user writes around it', async () => { + const T = newTable(); + await T.put({ id: 'a', name: 'one' }); + await T.writeReloadMarker(); + await T.put({ id: 'b', name: 'two' }); + + const groups = await Array.fromAsync( + await bridge.readAuditLog({ schema, table: T.tableName, search_type: 'timestamp', search_values: [0] }) + ); + assert.deepStrictEqual( + groups.map((group) => [group.operation, group.ids]), + [ + ['upsert', ['a']], + ['upsert', ['b']], + ] + ); + }); + + it('read_audit_log keeps a transaction group intact when the marker shares the transaction', async function () { + if (isLMDB) return this.skip(); // LMDB gives each write its own version, so nothing groups + const T = newTable(); + const context = {}; + await transaction(context, () => { + T.put({ id: 'a', name: 'one' }, context); + T.put({ id: 'b', name: 'two' }, context); + T.writeReloadMarker(context); + }); + await T.put({ id: 'c', name: 'three' }); + + const groups = await Array.fromAsync( + await bridge.readAuditLog({ schema, table: T.tableName, search_type: 'timestamp', search_values: [0] }) + ); + assert.deepStrictEqual( + groups.map((group) => group.ids), + [['a', 'b'], ['c']] + ); + assert.deepStrictEqual( + groups.flatMap((group) => group.records.map((record) => record.name)), + ['one', 'two', 'three'] + ); + }); + }); + + describe('a record whose primary entry has no local audit time', () => { + it('has an empty history, not an error', async function () { + if (!isLMDB) return this.skip(); // RocksDB resolves the audit position from the record's version + const T = newTable(); + // a write that bypasses the audit path, as a base copy does + T.primaryStore.putSync('orphan', { id: 'orphan', name: 'unaudited' }, Date.now()); + assert.strictEqual(T.primaryStore.getEntry('orphan').localTime, undefined); + + assert.deepStrictEqual(await T.getHistoryOfRecord('orphan'), []); + const byHash = await bridge.readAuditLog({ + schema, + table: T.tableName, + search_type: 'hash_value', + search_values: ['orphan'], + }); + assert.deepStrictEqual(byHash, { orphan: [] }); + await assert.rejects(T.getHistoryOfRecord(undefined), /An id is required/); + }); + }); +}); From 8ee8b32d8ade7c31c26d9de642608873d07b4007 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Wed, 28 Oct 2026 09:01:29 -0600 Subject: [PATCH 2/4] Say why replay and backfill do not use isNonHistoryType Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_016q5qJ8dVxeE9co9xaEPQtp Dispatch-Task: pr-maint-9c4f1f0aa5c37bd0cdb8f905b8b9ebea --- resources/auditStore.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/auditStore.ts b/resources/auditStore.ts index fce4a4d092..c34e4e1380 100644 --- a/resources/auditStore.ts +++ b/resources/auditStore.ts @@ -1383,7 +1383,7 @@ export function isLockControlType(type: unknown): boolean { return type === 'lockRelease' || type === 'lockBarrier'; } -/** Entry types that belong to no record's history: eviction, lock control and the whole-table `reload` marker. */ +/** Not for replay or backfill: those refuse or re-snapshot on a `reload` marker rather than skipping it. */ export function isNonHistoryType(type: unknown): boolean { return type === 'evict' || type === 'reload' || isLockControlType(type); } From 080ac88f6b2baa094cd5c93c4d4dad13a60185df Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Wed, 28 Oct 2026 09:08:45 -0600 Subject: [PATCH 3/4] Drop a test comment that restates the next line Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_016q5qJ8dVxeE9co9xaEPQtp Dispatch-Task: pr-maint-9c4f1f0aa5c37bd0cdb8f905b8b9ebea --- unitTests/resources/historyNonRecordEntries.test.js | 1 - 1 file changed, 1 deletion(-) diff --git a/unitTests/resources/historyNonRecordEntries.test.js b/unitTests/resources/historyNonRecordEntries.test.js index 1ddb73dd35..fd8e670cda 100644 --- a/unitTests/resources/historyNonRecordEntries.test.js +++ b/unitTests/resources/historyNonRecordEntries.test.js @@ -122,7 +122,6 @@ describe('history readers and non-record audit entries', function () { it('has an empty history, not an error', async function () { if (!isLMDB) return this.skip(); // RocksDB resolves the audit position from the record's version const T = newTable(); - // a write that bypasses the audit path, as a base copy does T.primaryStore.putSync('orphan', { id: 'orphan', name: 'unaudited' }, Date.now()); assert.strictEqual(T.primaryStore.getEntry('orphan').localTime, undefined); From 558594d1c97524f28a378fdabec11883abd16756 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Mon, 2 Nov 2026 10:39:54 -0700 Subject: [PATCH 4/4] Note the history-reader filter in audit-retention.md; resources/DESIGN.md has no byte headroom resources/DESIGN.md sits at its ceiling on main, so the note moves beside the existing `getHistory` bullet in audit-retention.md. Co-Authored-By: Claude Sonnet 5.5 Dispatch-Task: rebase-merge-kriszyp-harper-3494-5975580c897c --- resources/DESIGN.md | 2 -- resources/audit-retention.md | 5 +++++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/resources/DESIGN.md b/resources/DESIGN.md index 944ccdcd1a..b3df22bdb1 100644 --- a/resources/DESIGN.md +++ b/resources/DESIGN.md @@ -76,8 +76,6 @@ The synthesized target keeps the collection flag, or the batch is one null-keyed `getRecordAtTime` (`crdt.ts`) undoes only patches. Past a newer put, delete or invalidate it rebuilds forward (`reconstructForward`): carrying an image back returns a future state. A reversed field survives only when the reversal is exact. An unlinked put ends the chain without implying an empty past (`crdt.test.js`). -History readers skip entries that are not a record's history, through `isNonHistoryType` (`auditStore.ts`): `getHistory` and `getHistoryOfRecord` omit eviction, lock-control and the whole-table `reload` marker (`recordId: null`), so a marker never reaches `read_audit_log`. The `startTime` replay and the `previousCount` backfill must not use it: they handle a `reload` themselves (refuse or re-snapshot). A record with no audit position (an LMDB entry without `localTime`) has an empty history, like an absent record; carrying history across an audit-less base copy is harper#2212's design (`unitTests/resources/historyNonRecordEntries.test.js`). - ## Path routing & parameterised routes Parameterised routes sit outside the base map (`paramRoutes`), so exact and prefix matching stays fast and static routes win (`Resources.getMatch`). Literal > `:param` > `*`; a wildcard is last. `openApi.ts` and `components/mcp/resources.ts` each convert templates, so a syntax change touches both (`paramRoutes.test.js`). diff --git a/resources/audit-retention.md b/resources/audit-retention.md index 55406c54e3..721067a499 100644 --- a/resources/audit-retention.md +++ b/resources/audit-retention.md @@ -70,6 +70,11 @@ Things that are easy to get wrong here: - **`getHistory` is not in the floor's time domain.** The floor is an audit-log key (`subscribe`'s `localTime`); `getHistory` reports each entry's origin `version` under that name, which a backdated or replicated write makes differ, so its cursors cannot be compared against the floor. +- **History readers skip entries that describe no record's history.** `getHistory` and `getHistoryOfRecord` drop + `evict`, lock-control and `reload` entries through `isNonHistoryType` (`auditStore.ts`). Replay and the + `previousCount` backfill do not: they must see a `reload` to refuse or re-snapshot. A record with no audit + position (an LMDB entry without `localTime`) has an empty history, like an absent record; carrying history + across an audit-less base copy is harper#2212's design (`historyNonRecordEntries.test.js`). - **On RocksDB the floor tracks the configured retention horizon, not retained reality.** Whole-log-file purge granularity means the branch cannot know which entries a purge will drop, and the floor is written first, so each pass advances it to `Date.now() - auditRetention/(1+priority²)` whether a