diff --git a/resources/Table.ts b/resources/Table.ts index ec7e6def2a..14fe621a52 100644 --- a/resources/Table.ts +++ b/resources/Table.ts @@ -223,6 +223,7 @@ import { raiseAuditFloor, boundedAuditPruneEnd, isLockControlType, + isNonHistoryType, isAuditEntryWrite, getAuditResumeFloor, isResumablePosition, @@ -11151,8 +11152,7 @@ export function makeTable(options): TableResourceClass { end: endTime, })) { await rest(); // yield to other async operations - if (auditRecord.tableId !== tableId || auditRecord.type === 'evict' || isLockControlType(auditRecord.type)) - continue; + if (auditRecord.tableId !== tableId || isNonHistoryType(auditRecord.type)) continue; let value; try { value = auditRecord.getValue(primaryStore, true, auditRecord.txnLogKey); @@ -11182,7 +11182,7 @@ export function makeTable(options): TableResourceClass { let nextVersion = isRocksDB ? resolveAuditHead(id, entry.version, entry.nodeId, entry.additionalAuditRefs).txnLogKey : entry.localTime; - if (!nextVersion) throw new Error('The entry does not have a local audit time'); + if (!nextVersion) return history; const count = 0; const auditWindow = 100; let loggedEntryError = false; @@ -11194,8 +11194,7 @@ export function makeTable(options): TableResourceClass { for (const auditRecord of auditStore.getRange({ start, end: nextVersion + 0.001 })) { if ( auditRecord.tableId === tableId && - auditRecord.type !== 'evict' && - !isLockControlType(auditRecord.type) && + !isNonHistoryType(auditRecord.type) && compareKeys(auditRecord.recordId, id) === 0 ) { let value; diff --git a/resources/audit-retention.md b/resources/audit-retention.md index 55406c54e3..721067a499 100644 --- a/resources/audit-retention.md +++ b/resources/audit-retention.md @@ -70,6 +70,11 @@ Things that are easy to get wrong here: - **`getHistory` is not in the floor's time domain.** The floor is an audit-log key (`subscribe`'s `localTime`); `getHistory` reports each entry's origin `version` under that name, which a backdated or replicated write makes differ, so its cursors cannot be compared against the floor. +- **History readers skip entries that describe no record's history.** `getHistory` and `getHistoryOfRecord` drop + `evict`, lock-control and `reload` entries through `isNonHistoryType` (`auditStore.ts`). Replay and the + `previousCount` backfill do not: they must see a `reload` to refuse or re-snapshot. A record with no audit + position (an LMDB entry without `localTime`) has an empty history, like an absent record; carrying history + across an audit-less base copy is harper#2212's design (`historyNonRecordEntries.test.js`). - **On RocksDB the floor tracks the configured retention horizon, not retained reality.** Whole-log-file purge granularity means the branch cannot know which entries a purge will drop, and the floor is written first, so each pass advances it to `Date.now() - auditRetention/(1+priority²)` whether a diff --git a/resources/auditStore.ts b/resources/auditStore.ts index 35d3a526ad..c34e4e1380 100644 --- a/resources/auditStore.ts +++ b/resources/auditStore.ts @@ -1382,6 +1382,11 @@ function isDecodableAction(action: number) { export function isLockControlType(type: unknown): boolean { return type === 'lockRelease' || type === 'lockBarrier'; } + +/** Not for replay or backfill: those refuse or re-snapshot on a `reload` marker rather than skipping it. */ +export function isNonHistoryType(type: unknown): boolean { + return type === 'evict' || type === 'reload' || isLockControlType(type); +} const ORIGINATING_OPERATIONS = { insert: 1, update: 2, diff --git a/unitTests/resources/historyNonRecordEntries.test.js b/unitTests/resources/historyNonRecordEntries.test.js new file mode 100644 index 0000000000..fd8e670cda --- /dev/null +++ b/unitTests/resources/historyNonRecordEntries.test.js @@ -0,0 +1,139 @@ +const assert = require('assert'); +const { setupTestDBPath } = require('../testUtils'); +const { table } = require('#src/resources/databases'); +const { transaction } = require('#src/resources/transaction'); +const { setMainIsWorker } = require('#js/server/threads/manageThreads'); +const { ResourceBridge } = require('#src/dataLayer/harperBridge/ResourceBridge'); +require('#src/server/serverHelpers/serverUtilities'); + +const isLMDB = process.env.HARPER_STORAGE_ENGINE === 'lmdb'; +const schema = 'history_non_record_entries'; + +describe('history readers and non-record audit entries', function () { + this.timeout(60_000); + let tableIndex = 0; + const bridge = new ResourceBridge(); + + before(function () { + setupTestDBPath(); + setMainIsWorker(true); + }); + + function newTable(database = schema) { + return table({ + database, + table: `HistoryNonRecord${++tableIndex}`, + attributes: [{ name: 'id', isPrimaryKey: true }, { name: 'name' }], + }); + } + + function auditEntries(T) { + const entries = []; + for (const entry of T.auditStore.getRange({ start: 1 })) { + if (entry.tableId === T.tableId) entries.push({ type: entry.type, txnLogKey: entry.txnLogKey }); + } + return entries; + } + + describe('reload marker', () => { + it('getHistory and getHistoryOfRecord omit it', async () => { + const T = newTable(); + await T.put({ id: 'a', name: 'one' }); + await T.writeReloadMarker(); + await T.put({ id: 'a', name: 'two' }); + assert.ok( + auditEntries(T).some((entry) => entry.type === 'reload'), + 'the marker is in the audit log' + ); + + const history = await Array.fromAsync(T.getHistory()); + assert.deepStrictEqual( + history.map((entry) => [entry.type, entry.id]), + [ + ['put', 'a'], + ['put', 'a'], + ] + ); + const ofRecord = await T.getHistoryOfRecord('a'); + assert.deepStrictEqual( + ofRecord.map((entry) => entry.value.name), + ['one', 'two'] + ); + }); + + it('a table with only a marker has no history', async () => { + const T = newTable(); + await T.writeReloadMarker(); + assert.deepStrictEqual( + auditEntries(T).map((entry) => entry.type), + ['reload'] + ); + assert.deepStrictEqual(await Array.fromAsync(T.getHistory()), []); + const groups = await Array.fromAsync( + await bridge.readAuditLog({ schema, table: T.tableName, search_type: 'timestamp', search_values: [0] }) + ); + assert.deepStrictEqual(groups, []); + }); + + it('read_audit_log reports only the user writes around it', async () => { + const T = newTable(); + await T.put({ id: 'a', name: 'one' }); + await T.writeReloadMarker(); + await T.put({ id: 'b', name: 'two' }); + + const groups = await Array.fromAsync( + await bridge.readAuditLog({ schema, table: T.tableName, search_type: 'timestamp', search_values: [0] }) + ); + assert.deepStrictEqual( + groups.map((group) => [group.operation, group.ids]), + [ + ['upsert', ['a']], + ['upsert', ['b']], + ] + ); + }); + + it('read_audit_log keeps a transaction group intact when the marker shares the transaction', async function () { + if (isLMDB) return this.skip(); // LMDB gives each write its own version, so nothing groups + const T = newTable(); + const context = {}; + await transaction(context, () => { + T.put({ id: 'a', name: 'one' }, context); + T.put({ id: 'b', name: 'two' }, context); + T.writeReloadMarker(context); + }); + await T.put({ id: 'c', name: 'three' }); + + const groups = await Array.fromAsync( + await bridge.readAuditLog({ schema, table: T.tableName, search_type: 'timestamp', search_values: [0] }) + ); + assert.deepStrictEqual( + groups.map((group) => group.ids), + [['a', 'b'], ['c']] + ); + assert.deepStrictEqual( + groups.flatMap((group) => group.records.map((record) => record.name)), + ['one', 'two', 'three'] + ); + }); + }); + + describe('a record whose primary entry has no local audit time', () => { + it('has an empty history, not an error', async function () { + if (!isLMDB) return this.skip(); // RocksDB resolves the audit position from the record's version + const T = newTable(); + T.primaryStore.putSync('orphan', { id: 'orphan', name: 'unaudited' }, Date.now()); + assert.strictEqual(T.primaryStore.getEntry('orphan').localTime, undefined); + + assert.deepStrictEqual(await T.getHistoryOfRecord('orphan'), []); + const byHash = await bridge.readAuditLog({ + schema, + table: T.tableName, + search_type: 'hash_value', + search_values: ['orphan'], + }); + assert.deepStrictEqual(byHash, { orphan: [] }); + await assert.rejects(T.getHistoryOfRecord(undefined), /An id is required/); + }); + }); +});