Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 4 additions & 5 deletions resources/Table.ts
Original file line number Diff line number Diff line change
Expand Up @@ -223,6 +223,7 @@ import {
raiseAuditFloor,
boundedAuditPruneEnd,
isLockControlType,
isNonHistoryType,
isAuditEntryWrite,
getAuditResumeFloor,
isResumablePosition,
Expand Down Expand Up @@ -11151,8 +11152,7 @@ export function makeTable(options): TableResourceClass {
end: endTime,
})) {
await rest(); // yield to other async operations
if (auditRecord.tableId !== tableId || auditRecord.type === 'evict' || isLockControlType(auditRecord.type))
continue;
if (auditRecord.tableId !== tableId || isNonHistoryType(auditRecord.type)) continue;
let value;
try {
value = auditRecord.getValue(primaryStore, true, auditRecord.txnLogKey);
Expand Down Expand Up @@ -11182,7 +11182,7 @@ export function makeTable(options): TableResourceClass {
let nextVersion = isRocksDB
? resolveAuditHead(id, entry.version, entry.nodeId, entry.additionalAuditRefs).txnLogKey
: entry.localTime;
if (!nextVersion) throw new Error('The entry does not have a local audit time');
if (!nextVersion) return history;
const count = 0;
const auditWindow = 100;
let loggedEntryError = false;
Expand All @@ -11194,8 +11194,7 @@ export function makeTable(options): TableResourceClass {
for (const auditRecord of auditStore.getRange({ start, end: nextVersion + 0.001 })) {
if (
auditRecord.tableId === tableId &&
auditRecord.type !== 'evict' &&
!isLockControlType(auditRecord.type) &&
!isNonHistoryType(auditRecord.type) &&
compareKeys(auditRecord.recordId, id) === 0
) {
let value;
Expand Down
5 changes: 5 additions & 0 deletions resources/audit-retention.md
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,11 @@ Things that are easy to get wrong here:
- **`getHistory` is not in the floor's time domain.** The floor is an audit-log key (`subscribe`'s
`localTime`); `getHistory` reports each entry's origin `version` under that name, which a backdated
or replicated write makes differ, so its cursors cannot be compared against the floor.
- **History readers skip entries that describe no record's history.** `getHistory` and `getHistoryOfRecord` drop
`evict`, lock-control and `reload` entries through `isNonHistoryType` (`auditStore.ts`). Replay and the
`previousCount` backfill do not: they must see a `reload` to refuse or re-snapshot. A record with no audit
position (an LMDB entry without `localTime`) has an empty history, like an absent record; carrying history
across an audit-less base copy is harper#2212's design (`historyNonRecordEntries.test.js`).
- **On RocksDB the floor tracks the configured retention horizon, not retained reality.** Whole-log-file
purge granularity means the branch cannot know which entries a purge will drop, and the floor is
written first, so each pass advances it to `Date.now() - auditRetention/(1+priorityΒ²)` whether a
Expand Down
5 changes: 5 additions & 0 deletions resources/auditStore.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1382,6 +1382,11 @@ function isDecodableAction(action: number) {
export function isLockControlType(type: unknown): boolean {
return type === 'lockRelease' || type === 'lockBarrier';
}

/** Not for replay or backfill: those refuse or re-snapshot on a `reload` marker rather than skipping it. */
export function isNonHistoryType(type: unknown): boolean {
return type === 'evict' || type === 'reload' || isLockControlType(type);
}
const ORIGINATING_OPERATIONS = {
insert: 1,
update: 2,
Expand Down
139 changes: 139 additions & 0 deletions unitTests/resources/historyNonRecordEntries.test.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,139 @@
const assert = require('assert');
const { setupTestDBPath } = require('../testUtils');
const { table } = require('#src/resources/databases');
const { transaction } = require('#src/resources/transaction');
const { setMainIsWorker } = require('#js/server/threads/manageThreads');
const { ResourceBridge } = require('#src/dataLayer/harperBridge/ResourceBridge');
require('#src/server/serverHelpers/serverUtilities');

const isLMDB = process.env.HARPER_STORAGE_ENGINE === 'lmdb';
const schema = 'history_non_record_entries';

describe('history readers and non-record audit entries', function () {
this.timeout(60_000);
let tableIndex = 0;
const bridge = new ResourceBridge();

before(function () {
setupTestDBPath();
setMainIsWorker(true);
});

function newTable(database = schema) {
return table({
database,
table: `HistoryNonRecord${++tableIndex}`,
attributes: [{ name: 'id', isPrimaryKey: true }, { name: 'name' }],
});
}

function auditEntries(T) {
const entries = [];
for (const entry of T.auditStore.getRange({ start: 1 })) {
if (entry.tableId === T.tableId) entries.push({ type: entry.type, txnLogKey: entry.txnLogKey });
}
return entries;
}

describe('reload marker', () => {
it('getHistory and getHistoryOfRecord omit it', async () => {
const T = newTable();
await T.put({ id: 'a', name: 'one' });
await T.writeReloadMarker();
await T.put({ id: 'a', name: 'two' });
assert.ok(
auditEntries(T).some((entry) => entry.type === 'reload'),
'the marker is in the audit log'
);

const history = await Array.fromAsync(T.getHistory());
assert.deepStrictEqual(
history.map((entry) => [entry.type, entry.id]),
[
['put', 'a'],
['put', 'a'],
]
);
const ofRecord = await T.getHistoryOfRecord('a');
assert.deepStrictEqual(
ofRecord.map((entry) => entry.value.name),
['one', 'two']
);
});

it('a table with only a marker has no history', async () => {
const T = newTable();
await T.writeReloadMarker();
assert.deepStrictEqual(
auditEntries(T).map((entry) => entry.type),
['reload']
);
assert.deepStrictEqual(await Array.fromAsync(T.getHistory()), []);
const groups = await Array.fromAsync(
await bridge.readAuditLog({ schema, table: T.tableName, search_type: 'timestamp', search_values: [0] })
);
assert.deepStrictEqual(groups, []);
});

it('read_audit_log reports only the user writes around it', async () => {
const T = newTable();
await T.put({ id: 'a', name: 'one' });
await T.writeReloadMarker();
await T.put({ id: 'b', name: 'two' });

const groups = await Array.fromAsync(
await bridge.readAuditLog({ schema, table: T.tableName, search_type: 'timestamp', search_values: [0] })
);
assert.deepStrictEqual(
groups.map((group) => [group.operation, group.ids]),
[
['upsert', ['a']],
['upsert', ['b']],
]
);
});

it('read_audit_log keeps a transaction group intact when the marker shares the transaction', async function () {
if (isLMDB) return this.skip(); // LMDB gives each write its own version, so nothing groups
const T = newTable();
const context = {};
await transaction(context, () => {
T.put({ id: 'a', name: 'one' }, context);
T.put({ id: 'b', name: 'two' }, context);
T.writeReloadMarker(context);
});
await T.put({ id: 'c', name: 'three' });

const groups = await Array.fromAsync(
await bridge.readAuditLog({ schema, table: T.tableName, search_type: 'timestamp', search_values: [0] })
);
assert.deepStrictEqual(
groups.map((group) => group.ids),
[['a', 'b'], ['c']]
);
assert.deepStrictEqual(
groups.flatMap((group) => group.records.map((record) => record.name)),
['one', 'two', 'three']
);
});
});

describe('a record whose primary entry has no local audit time', () => {
it('has an empty history, not an error', async function () {
if (!isLMDB) return this.skip(); // RocksDB resolves the audit position from the record's version
const T = newTable();
T.primaryStore.putSync('orphan', { id: 'orphan', name: 'unaudited' }, Date.now());
assert.strictEqual(T.primaryStore.getEntry('orphan').localTime, undefined);

assert.deepStrictEqual(await T.getHistoryOfRecord('orphan'), []);
const byHash = await bridge.readAuditLog({
schema,
table: T.tableName,
search_type: 'hash_value',
search_values: ['orphan'],
});
assert.deepStrictEqual(byHash, { orphan: [] });
await assert.rejects(T.getHistoryOfRecord(undefined), /An id is required/);
});
});
});
Loading