diff --git a/package.json b/package.json index 5dfc1721cc..738460c5a3 100644 --- a/package.json +++ b/package.json @@ -80,7 +80,7 @@ "test:unit:resources": "mocha \"unitTests/resources/**/*.js\" --exclude \"unitTests/resources/**/*.bench.js\"", "test:unit:resources:core": "mocha \"unitTests/resources/**/*.js\" --exclude \"unitTests/resources/**/*.bench.js\" --exclude \"unitTests/resources/**/*{[Ii]ndex,[Hh]nsw,[Vv]ector}*.js\"", "test:unit:resources:indexes": "mocha \"unitTests/resources/**/*{[Ii]ndex,[Hh]nsw,[Vv]ector}*.js\" --exclude \"unitTests/resources/**/*.bench.js\"", - "bench": "mocha \"unitTests/resources/caching-rocks-database.bench.js\" \"unitTests/resources/indexRangeSelect.bench.js\" \"unitTests/resources/originClosedFloor.bench.js\" \"unitTests/resources/liveSubscriptionReauth.bench.js\"", + "bench": "mocha \"unitTests/resources/caching-rocks-database.bench.js\" \"unitTests/resources/indexRangeSelect.bench.js\" \"unitTests/resources/originClosedFloor.bench.js\" \"unitTests/resources/subscriptionCheckpoint.bench.js\" \"unitTests/resources/liveSubscriptionReauth.bench.js\"", "test:unit:bin": "mocha \"unitTests/bin/**/*.js\"", "test:unit:apitests": "node ./dist/bin/harper.js stop && mocha \"unitTests/apiTests/**/*-test.mjs\"", "test:unit:logging": "mocha \"unitTests/utility/logging/*.js\"", diff --git a/resources/DESIGN.md b/resources/DESIGN.md index d3ddf8e433..ed79a100d8 100644 --- a/resources/DESIGN.md +++ b/resources/DESIGN.md @@ -473,7 +473,7 @@ A replication receiver feeds one source subscription per database from every con A transaction-log entry's identity is (`txnLogKey`, `nodeId`): the key alone repeats across origins, and a receiver files a replicated transaction under the origin's key. `RocksTransactionLogStore.put` therefore files an origin's entries in that origin's own log, created on first use, never in the relaying peer's log, so a key stays unique within a log (exact resume, `resumePastExactStart`, depends on it); within `local` it is unique because every key is minted there ("Origin-closed timestamp floor" above). A relayed entry whose origin id has no node name is rejected rather than misfiled; an unnamed id that was not relayed (`viaNodeId` equal to `nodeId` or absent) is not a replication origin and stays in `local`, under a minted key. -A failed transaction on a tagged stream (a non-retryable commit failure, including a segment closed by a later `beginTxn`) is reported at that stream's next `end_txn`: when the `end_txn` carries `onFailure(error, txnLogKey)`, neither its `onCommit` nor its sequence id passes the failure, and `onFailure` returning true _holds_ the stream — every later `end_txn` on it commits its writes without `onCommit` or a cursor, because the source is reconnecting to replay from its durable cursor under a new token (a held token is never reused). A held failure is not reported to the apply-failure listeners, since the replay applies it; a failure the source moves past (`onFailure` false, or no `onFailure`) is. An `end_txn` without `onFailure` keeps the older behavior of recording its cursor, so holding is opt-in per source. `txnStream` must be an object: streams are held in a `WeakMap`, so a closed connection's state is collected with its token. +A failed transaction on a tagged stream (a non-retryable commit failure, including a segment closed by a later `beginTxn`) is reported at that stream's next `end_txn`: when the `end_txn` carries `onFailure(error, txnLogKey)`, neither its `onCommit` nor its sequence id passes the failure, and `onFailure` returning true _holds_ the stream — every later `end_txn` on it commits its writes without `onCommit` or a cursor, because the source is reconnecting to replay from its durable cursor under a new token (a held token is never reused). A held failure is not reported to the apply-failure listeners, since the replay applies it; a failure the source moves past (`onFailure` false, or no `onFailure`) is. An `end_txn` without `onFailure` keeps the older behavior of recording its cursor, so holding is opt-in per source. When the transaction committed, and any `onCommit` ran, but the sequence-row write itself failed, `onFailure(error, txnLogKey, true)` runs before the error propagates. The third argument tells the source that the commit path completed, so nothing it releases on commit is still held; only what it attached for recording (cursors, floors, route state) was lost. The stream's last sequence id is rolled back, so a repeat of that position writes again. `txnStream` must be an object: streams are held in a `WeakMap`, so a closed connection's state is collected with its token. An `end_txn` can also carry `originCursors: [nodeId, originLogKey][]`, the highest origin log key the source has durably applied for each origin. The apply loop merges them, keeping the max, into the connection's `[seq, remoteNodeIds[0]]` row as `nodes[].originLogKey`, even when `localTime` did not move (harper-pro#989; the per-origin cursor vector of harper-pro's replication workstream W4, harper-pro#434). A source may attach the array from `onCommit`, and the loop reads it only after awaiting `onCommit`. A held stream records neither the sequence id, origin cursors nor origin floors. A source that sends origin cursors must tag its stream and pass `onFailure`: without them, a failed segment's later `end_txn` still merges its cursor vector, so a cursor would pass the failed transaction; floors from such a source are dropped (below). The same `end_txn` can carry `originFloors: [nodeId, closedFloor, relayable][]` — an origin's closed timestamp floor (harper-pro#922 item 2; the producer is "Origin-closed timestamp floor" above), merged by max into `nodes[].closedFloor` with `nodes[].relayable` (whether the source received that origin directly with full table coverage, so a relay may forward the floor). It is kept apart from `originLogKey`: an applied position can sit above a transaction still open at the origin, a floor cannot, so only the floor is proof that resume may start there. A floor-only `end_txn` writes the row like a cursor-only one. Floors are merged only from a tagged stream that passes `onFailure` — core enforces for floors what it only documents for cursors, since a stored floor is what resume trusts — and an entry whose floor is not a positive finite number is skipped. @@ -638,7 +638,8 @@ A primary key declared on an existing keyless table cannot re-key its records. S A `reportProgress` subscription on RocksDB exposes `checkpoint(holds?)`: the issue's compact form `{ v, scope, time, overlap, exceptions, members }`, where `S = time - overlap` is the inclusive replay boundary of every member not in `exceptions` and `members` is the XOR of `sha256(origin, originGeneration)` over every member, so healthy origins are neither named nor counted while the set is still bound. The per-origin vector stays internal: `B_O = min(F_O, oldest event of O handed to the queue and not yet taken, holds[O])`, encoded with `time = max B`, `S = max(min B, time - MAX_OVERLAP_MS)`; lowering any boundary only adds duplicates. `F_O` is certified by the broadcaster (`startOriginTracking`): every 5 s it captures the qualifying floors (`captureCertifiedFloors`: the own published floor, and every seq-row origin with a level-2 `closedFloor`, `originGeneration`, `fullCoverage`, `certifiedFrom`, no `partialHistory`, `hole` or `floorRevoked`) and schedules a notify pass; the capture a fresh pass began with (`passCapture`) is published as `certifiedFloors` only when that pass drains without a failed read, since every entry below a captured floor was in the log before the capture and the drained pass dispatched it. **Strict availability:** one known origin (own, any seq row, any log) without a qualifying floor means no capture, so `checkpoint()` is undefined and MQTT keeps today's scalar. -- **Admission** (`admitCheckpoint`, `subscribe({ resume })`, 400/409/410): the scope string must match (database, table, table generation, target, delivery mode, `filterKey`); membership is resolved by subtracting at most four candidates whose route here began at the origin's first retained entry (`certifiedFrom === 0`, log retains sequence 1 — a node that joined after the checkpoint, admitted at boundary 0) from this node's candidates, at most 100 000 combinations; every member boundary must be at or above the resume floor, its origin not revoked, and held here by a full-coverage route with `certifiedFrom ≤ B_O`. Admission facts are rechecked at each replay yield against `auditStore.originStateEpoch`, which the seq-row merge bumps on any qualifier change. The replay seeks each member's log inclusively (`startByLog`, snapshot false, `includeLogName`), a record target filtered by exact key identity, and the subscription is `ungated`: the broadcaster's `startTime >= key` gate would drop a late lower-key commit, the omission harper#2928 describes. The boundary during the replay is the token's; after it, the certified floors (every entry below one appended before the replay ended was read by it, or dispatched live). **Qualifier lifetimes** (the seq-row merge): a revocation (a downgrade, or an applied-below-floor report) and a hole are sticky within an origin generation; a certificate naming a new origin generation starts that origin over (floor, revocation and `certifiedFrom` cleared, so the route proves its prefix again), and a completed base copy clears `partialHistory` and `hole`. `certifiedFrom` is the start of the route that certified the floor (two whole routes can cover disjoint intervals, so another row's start proves nothing about this floor), and a whole-log replay (a late joiner, a zero boundary) needs that same route to have begun at the origin's first entry, since another whole route can cover a disjoint interval; of two generations of one origin the one whose rows saw the highest floor is current (floors sample the origin's clock; a revoked row keeps the floor it was revoked at; a removed peer's retained row is frozen at its death), and the floor, revocations and holes count only from rows of that generation; a queued event without an origin tag holds every boundary, except a snapshot row, which is state rather than a log position; a repeated copy-completion mark clears a hole whether or not `partialHistory` changed; a certificate without a generation on a row that had one clears its level-2 qualifiers (that route certifies at level 1 now). A certified subscription that resumed from a scalar position delivers twice what its replay read and the live dispatch runs after it went live, and never what committed below its position before then: a scalar resume keeps main's exposure. A route mark or revocation names the generation it was derived under and is ignored once the row's generation moved on. Revocations and holes count only from rows of the current generation, so a dead peer's row for an earlier generation cannot hold the origin back. The epoch lives in a shared buffer (`bumpOriginStateEpoch`, after the row commits), so a recheck on any thread sees it. Every certified subscription is `ungated` once live, not only a checkpoint resume: the scalar-resumed MQTT session a rolling upgrade carries would otherwise drop a late lower-key commit the next floor certifies as delivered. +- **Admission** (`admitCheckpoint`, `subscribe({ resume })`, 400/409/410): the scope string must match (database, table, table generation, target, delivery mode, `filterKey`); membership is resolved by subtracting at most four candidates whose route here began at the origin's first retained entry (`certifiedFrom === 0`, log retains sequence 1 — a node that joined after the checkpoint, admitted at boundary 0) from this node's candidates, at most 100 000 combinations; every member boundary must be at or above the resume floor, its origin not revoked, and held here by a full-coverage route with `certifiedFrom ≤ B_O`. Admission facts are rechecked at each replay yield against `auditStore.originStateEpoch`, which the seq-row merge bumps on any qualifier change and a copy-bound writer bumps after its commit. The replay seeks each member's log inclusively (`startByLog`, snapshot false, `includeLogName`), a record target filtered by exact key identity, and the subscription is `ungated`: the broadcaster's `startTime >= key` gate would drop a late lower-key commit, the omission harper#2928 describes. The boundary during the replay is the token's; after it, the certified floors (every entry below one appended before the replay ended was read by it, or dispatched live). **Qualifier lifetimes** (the seq-row merge): a revocation (a downgrade, or an applied-below-floor report) and a hole are sticky within an origin generation; a certificate naming a new origin generation starts that origin over (floor, revocation and `certifiedFrom` cleared, so the route proves its prefix again), and a completed base copy clears `partialHistory` and `hole`. `certifiedFrom` is the start of the route that certified the floor (two whole routes can cover disjoint intervals, so another row's start proves nothing about this floor), and a whole-log replay (a late joiner, a zero boundary) needs that same route to have begun at the origin's first entry, since another whole route can cover a disjoint interval; of two generations of one origin the one whose rows saw the highest floor is current (floors sample the origin's clock; a revoked row keeps the floor it was revoked at; a removed peer's retained row is frozen at its death), and the floor, revocations and holes count only from rows of that generation; a queued event without an origin tag holds every boundary, except a snapshot row, which is state rather than a log position; a repeated copy-completion mark clears a hole whether or not `partialHistory` changed; a certificate without a generation on a row that had one clears its level-2 qualifiers (that route certifies at level 1 now). A certified subscription that resumed from a scalar position delivers twice what its replay read and the live dispatch runs after it went live, and never what committed below its position before then: a scalar resume keeps main's exposure. A route mark or revocation names the generation it was derived under and is ignored once the row's generation moved on. Revocations and holes count only from rows of the current generation, so a dead peer's row for an earlier generation cannot hold the origin back. The epoch lives in a shared buffer (`bumpOriginStateEpoch`, after the row commits), so a recheck on any thread sees it. Every certified subscription is `ungated` once live, not only a checkpoint resume: the scalar-resumed MQTT session a rolling upgrade carries would otherwise drop a late lower-key commit the next floor certifies as delivered. +- **One writer per `[seq, P]` row; copy bounds are applied by the reader.** `updateRecordedSequenceId`, in route P's apply loop, is the only writer of `[seq, P]`: it reads the row and commits it from that snapshot with no cross-thread lock, so any other writer's change can be overwritten (rocksdb-js `occValidation: 'parallel'` does not flag a plain put landing between a commit's conflict check and its write). A source changes its own row only through `end_txn` fields. Route state (`originRouteStates`) is `[id, certifiedFrom, partialHistory, hole, generation?, fullCoverage?]`; `fullCoverage: true` restores a level-2 route's coverage after a copy that ran with partial coverage. A revocation naming no generation against a node with none clears its level-1 floor rather than revoking it, since there is no generation for the revocation to end with. An audit-less base copy (`isCopyApply`) leaves this node's log of each copied origin incomplete below the highest version it wrote, a database-wide fact a route row cannot carry, because the route that certified may be disconnected and has no writer. So the copying source records that version per origin in its own `[copyBound, peerId]` row (`{ [originId]: version }`, committed before the copied row is applied), and `readOriginStates` treats a route's `certifiedFrom` at or below any peer's bound for that origin as absent (`readCopyBounds`; a malformed row throws a 410, so admission refuses and capture issues nothing). A bound for this node's own origin (a copy restoring its own lost rows) raises the own start above it, so an own boundary at or below it, or a whole-log replay, is refused. A bound commit bumps the origin-state epoch, and a capture carries the epoch it read before its scans: `certifiedFloors` drops a capture the epoch has since passed, and an admitted replay rechecks. The two halves ship together: a core without the bound reader admits a route whose start a copy has since voided. Pinned by `subscribeCheckpoint.test.js` and `replicationSeqCursor.test.js`. - **Not covered:** SSE/WS resume (no transport exposes a token; harper#2448 deferred them too); an origin two hops away (one-hop relay) or reached only over a partial route never qualifies; `eventFilter`/`rowFilter` without `filterKey` issues no token; the own floor qualifies only on an engine with `syncWal`. Pinned by `subscribeCheckpoint.test.js` and `durableSessionCheckpoint.test.js`. ## Subscription version selection (`Table.ts`) diff --git a/resources/Table.ts b/resources/Table.ts index ff7d586c37..fa94960212 100644 --- a/resources/Table.ts +++ b/resources/Table.ts @@ -2240,9 +2240,9 @@ export function makeTable(options): TableResourceClass { const originFloorRevocations: (number | [number, string?])[] | undefined = certifyingStream ? event.originFloorRevocations : undefined; - // `[id, certifiedFrom, partialHistory, hole, generation?]`: a mark derived under one origin generation - // says nothing about another, so a generation it names must be the row's - const originRouteStates: [number, number | undefined, boolean, boolean, string?][] | undefined = + // `[id, certifiedFrom, partialHistory, hole, generation?, fullCoverage?]`: a mark derived under one origin + // generation says nothing about another, so a generation it names must be the row's + const originRouteStates: [number, number | undefined, boolean, boolean, string?, boolean?][] | undefined = certifyingStream ? event.originRouteStates : undefined; let qualifiersChanged = false; // the key for tracking the sequence ids and txn times received from this node @@ -2344,6 +2344,15 @@ export function makeTable(options): TableResourceClass { if (!nodeState) nodeStates.push((nodeState = { id: nodeId })); if (nodeState.floorRevoked === true) continue; if (generation !== undefined && nodeState.originGeneration !== generation) continue; + if (generation === undefined && typeof nodeState.originGeneration !== 'string') { + // a level-1 floor has no generation to stay revoked within: clear it, and a later certificate floors again + if (nodeState.closedFloor !== undefined || nodeState.relayable) { + nodeState.closedFloor = undefined; + nodeState.relayable = false; + originCursorsChanged = true; + } + continue; + } nodeState.floorRevoked = true; // the floor it was revoked at still dates this generation against another's rows if (isValidFloor(nodeState.closedFloor)) nodeState.revokedFloor = nodeState.closedFloor; @@ -2352,7 +2361,14 @@ export function makeTable(options): TableResourceClass { qualifiersChanged = true; } if (originRouteStates) - for (const [nodeId, certifiedFrom, partialHistory, hole, generation] of originRouteStates) { + for (const [ + nodeId, + certifiedFrom, + partialHistory, + hole, + generation, + fullCoverage, + ] of originRouteStates) { let nodeState = nodeStates.find((existingNode) => existingNode.id === nodeId); if (!nodeState) nodeStates.push((nodeState = { id: nodeId })); if (generation !== undefined && nodeState.originGeneration !== generation) continue; @@ -2383,6 +2399,14 @@ export function makeTable(options): TableResourceClass { nodeState.hole = true; qualifiersChanged = true; } + if ( + fullCoverage === true && + typeof nodeState.originGeneration === 'string' && + nodeState.fullCoverage === false + ) { + nodeState.fullCoverage = true; + qualifiersChanged = true; + } } if (qualifiersChanged) originCursorsChanged = true; const seqId = Math.max(existingSeq?.seqId ?? 1, event.localTime || 0); @@ -2523,6 +2547,7 @@ export function makeTable(options): TableResourceClass { continue; } const advancesSequence = !!event.localTime && stream.lastSequenceId !== event.localTime; + const previousSequenceId = stream.lastSequenceId; if (advancesSequence && event.remoteNodeIds?.length > 0) stream.lastSequenceId = event.localTime; // Backpressure: wait for the transaction's commit to land before recording the sequence // id or pulling the next event. This serializes the apply loop so bulk ingest can't @@ -2586,7 +2611,15 @@ export function makeTable(options): TableResourceClass { event.originRouteStates?.length > 0 || event.originFloorRevocations?.length > 0) ) - await updateRecordedSequenceId(event, advancesSequence, committingTxn?.nodeId); + try { + await updateRecordedSequenceId(event, advancesSequence, committingTxn?.nodeId); + } catch (error) { + // the transaction landed but its position and route state did not: a repeat of this position + // must write again, and the source that attached them must learn it, or it counts them as recorded + stream.lastSequenceId = previousSequenceId; + if (event.onFailure && (await event.onFailure(error, failurePosition, true))) stream.held = true; + throw error; + } continue; } if (txnInProgress) { diff --git a/resources/subscriptionCheckpoint.ts b/resources/subscriptionCheckpoint.ts index fa2c4b8604..97343edbea 100644 --- a/resources/subscriptionCheckpoint.ts +++ b/resources/subscriptionCheckpoint.ts @@ -23,6 +23,7 @@ export interface SubscriptionCheckpoint { /** The event-origin tag a progress-reporting subscription carries on each event: the origin's local node id. */ export const ORIGIN_ID = Symbol.for('subscription-origin'); const SEQ_KEY = Symbol.for('seq'); +export const COPY_BOUND_KEY = Symbol.for('copyBound'); const EPOCH_BUFFER = 'origin-state-epoch'; const epochWords = new WeakMap(); @@ -98,6 +99,34 @@ export function membersDigest(members: Iterable<[name: string, generation: strin return digest.toString('hex'); } +/** + * The highest record version an audit-less base copy wrote here, per origin, over every copying peer's + * `[copyBound, peerId]` row (`{ [localOriginId]: version }`, written only by that peer's copy). This node's log of + * the origin lacks those rows, so no route certified from at or below the bound holds the origin complete. A + * malformed row throws: a bound read as absent would admit. + */ +export function readCopyBounds(dbisDb: any): Map { + const bounds = new Map(); + for (const { value } of dbisDb.getRange({ start: COPY_BOUND_KEY, end: [COPY_BOUND_KEY, Buffer.from([0xff])] })) { + const prototype = value && typeof value === 'object' ? Object.getPrototypeOf(value) : undefined; + if (prototype !== Object.prototype && prototype !== null) throw unreadableCopyBound(); + for (const key in value) { + const originId = Number(key); + const bound = value[key]; + if (!(Number.isInteger(originId) && originId >= 0 && String(originId) === key)) throw unreadableCopyBound(); + if (!(typeof bound === 'number' && Number.isFinite(bound) && bound >= 0)) throw unreadableCopyBound(); + if (!(bounds.get(originId)! >= bound)) bounds.set(originId, bound); + } + } + return bounds; +} + +function unreadableCopyBound(): ResumeHistoryUnavailableError { + return new ResumeHistoryUnavailableError( + 'A copy bound record is unreadable, so no checkpoint can be shown complete; resubscribe to resynchronize' + ); +} + /** Reads every `[seq, peerId]` row and merges the checkpoint-grade state of each origin it names. */ function readOriginStates(auditStore: any, dbisDb: any): Map { const states = new Map(); @@ -116,6 +145,20 @@ function readOriginStates(auditStore: any, dbisDb: any): Map + isValidFloor(node.closedFloor) && + typeof node.originGeneration === 'string' && + node.fullCoverage === true && + node.partialHistory !== true && + node.hole !== true && + node.floorRevoked !== true && + typeof node.certifiedFrom === 'number' && + !(node.certifiedFrom > node.closedFloor) && + !(node.certifiedFrom <= copyBounds.get(node.id)!); for (const { key, value } of dbisDb.getRange({ start: SEQ_KEY, end: [SEQ_KEY, Buffer.from([0xff])] })) { const nodes = value?.nodes; if (!Array.isArray(nodes) || typeof key?.[1] !== 'number') continue; @@ -130,16 +173,7 @@ function readOriginStates(auditStore: any, dbisDb: any): Map= node.closedFloor) - ) { + if (!(state.floor >= node.closedFloor) && routeCertifies(node)) { // the route that certified the floor is the one whose prefix below it is complete: another whole // route's start says nothing about this one, since their coverage can be disjoint state.floor = node.closedFloor; @@ -173,15 +207,7 @@ function readOriginStates(auditStore: any, dbisDb: any): Map= node.closedFloor) - ) { + if (!(state.floor >= node.closedFloor) && routeCertifies(node)) { state.floor = node.closedFloor; state.certifiedFrom = node.certifiedFrom; state.fullCoverage = true; @@ -206,7 +232,8 @@ function qualifies(state: OriginState): boolean { state.partialHistory !== true && state.hole !== true && state.floorRevoked !== true && - typeof state.certifiedFrom === 'number' + typeof state.certifiedFrom === 'number' && + !(state.certifiedFrom > state.floor!) ); } @@ -223,6 +250,8 @@ export interface CertifiedFloors { /** By local origin node id. */ origins: Map; capturedAt: number; + /** The origin-state epoch read before the capture: a qualifier or copy bound committed since may void it. */ + epoch: number; } /** @@ -230,13 +259,14 @@ export interface CertifiedFloors { * a qualifying floor, or nothing is captured. */ export function captureCertifiedFloors(auditStore: any, dbisDb: any): CertifiedFloors | undefined { + const epoch = readOriginStateEpoch(auditStore.rootStore); const states = readOriginStates(auditStore, dbisDb); if (states.size > MAX_ORIGINS) return; for (const state of states.values()) if (!qualifies(state)) return; const byName = new Map(); for (const state of states.values()) byName.set(state.name, state); for (const name of knownLogNames(auditStore)) if (!byName.has(name)) return; - return { origins: states, capturedAt: Date.now() }; + return { origins: states, capturedAt: Date.now(), epoch }; } const KEY_SCRATCH = new Float64Array(1); @@ -250,6 +280,13 @@ export function positionBefore(key: number): number { return KEY_SCRATCH[0]; } +function positionAfter(key: number): number { + if (key === 0) return Number.MIN_VALUE; + KEY_SCRATCH[0] = key; + KEY_BITS[0] += 1n; + return KEY_SCRATCH[0]; +} + /** Folds per-origin boundaries into the compact form; a boundary is never raised. */ export function encodeCheckpoint(scope: string, boundaries: Map): SubscriptionCheckpoint { let time = 0; @@ -455,6 +492,11 @@ export function admitCheckpoint( `Origin ${name} may have written below its certified floor; resubscribe to resynchronize` ); } + if (state.nodeId === 0 && !(state.certifiedFrom! <= boundary)) { + throw new ResumeHistoryUnavailableError( + `A copy restored rows of this node's own origin above the checkpoint boundary; resubscribe to resynchronize` + ); + } if (state.nodeId !== 0 && !(qualifies(state) && state.certifiedFrom <= boundary)) { throw new ResumeHistoryUnavailableError( `This node holds no complete route for origin ${name} from the checkpoint boundary; resubscribe to resynchronize` diff --git a/resources/transactionBroadcast.ts b/resources/transactionBroadcast.ts index 16efaaa6e9..4a536e9de6 100644 --- a/resources/transactionBroadcast.ts +++ b/resources/transactionBroadcast.ts @@ -3,7 +3,12 @@ import { warn } from '../utility/logging/harper_logger.ts'; import { DatabaseClosingError, DatabaseGenerationChangedError } from '../utility/errors/hdbError.ts'; import { IterableEventQueue } from './IterableEventQueue.ts'; import { ORIGIN_FLOOR_TICK_MS, localStartBound } from './originClosedFloor.ts'; -import { captureCertifiedFloors, positionBefore, type CertifiedFloors } from './subscriptionCheckpoint.ts'; +import { + captureCertifiedFloors, + positionBefore, + readOriginStateEpoch, + type CertifiedFloors, +} from './subscriptionCheckpoint.ts'; import { keyArrayToString } from './Resources.ts'; import type { Id } from './ResourceInterface.ts'; @@ -199,7 +204,9 @@ function stopOriginTracking(databaseSubscriptions: any) { export function certifiedFloors(subscription: any): CertifiedFloors | undefined { const databaseSubscriptions = subscription.subscriptions?.tables?.envs; if (!subscription.reportsProgress || !databaseSubscriptions || databaseSubscriptions.progressStopped) return; - return databaseSubscriptions.certifiedFloors ?? undefined; + const floors: CertifiedFloors | undefined = databaseSubscriptions.certifiedFloors ?? undefined; + if (floors && floors.epoch !== readOriginStateEpoch(databaseSubscriptions.auditStore?.rootStore)) return; + return floors; } function rangeFailures(range: any): number { diff --git a/unitTests/resources/replicationSeqCursor.test.js b/unitTests/resources/replicationSeqCursor.test.js index 9d481198d6..ed850023bf 100644 --- a/unitTests/resources/replicationSeqCursor.test.js +++ b/unitTests/resources/replicationSeqCursor.test.js @@ -153,6 +153,66 @@ describe('replication sequence-cursor write (harper-pro#603)', () => { } }); + it('reports a failed cursor commit to the end_txn that carried it, so its source re-sends what it attached', async function () { + let release; + const held = new Promise((resolve) => (release = resolve)); + const now = Date.now(); + const txnStream = {}; + const failures = []; + const ReplicatedTable = makeReplicatedTable( + 'SeqCursorFailureReportTable', + [ + { type: 'put', id: 1, value: { id: 1, name: 'first' }, timestamp: now }, + { + type: 'end_txn', + localTime: now, + timestamp: now, + remoteNodeIds: [47], + txnStream, + onFailure: (error, position, afterCommit) => { + failures.push({ error, afterCommit }); + return false; + }, + }, + { type: 'put', id: 3, value: { id: 3, name: 'repeat' }, timestamp: now }, + { type: 'end_txn', localTime: now, timestamp: now, remoteNodeIds: [47], txnStream, onFailure: () => false }, + { type: 'put', id: 2, value: { id: 2, name: 'second' }, timestamp: now + 1 }, + { + type: 'end_txn', + localTime: now + 1, + timestamp: now + 1, + remoteNodeIds: [47], + txnStream, + onFailure: () => false, + }, + ], + held + ); + let failNext = true; + const spy = spyOnCursorWrites(ReplicatedTable, (transaction) => { + if (!failNext) return; + failNext = false; + transaction.commit = () => Promise.reject(new Error('forced cursor commit failure')); + }); + try { + await waitFor(() => readCursor(ReplicatedTable, 47)?.seqId === now + 1, { + timeout: 5000, + message: 'the next cursor still records', + }); + assert.equal( + spy.staged.filter(({ value }) => value.seqId === now).length, + 2, + 'the repeated position wrote again' + ); + assert.equal(failures.length, 1); + assert.match(failures[0].error.message, /forced cursor commit failure/); + assert.equal(failures[0].afterCommit, true, 'onCommit already ran, so the source must not release twice'); + } finally { + spy.restore(); + release(); + } + }); + it('merges per-origin cursors attached at commit, even when the sequence does not advance', async function () { let release; const held = new Promise((resolve) => (release = resolve)); @@ -332,6 +392,79 @@ describe('replication sequence-cursor write (harper-pro#603)', () => { } }); + it('clears a level-1 floor on a genless revocation and restores a level-2 route coverage from a route mark', async function () { + let release; + const held = new Promise((resolve) => (release = resolve)); + const now = Date.now(); + const txnStream = {}; + const update = (fields) => ({ + type: 'end_txn', + localTime: now, + timestamp: now, + remoteNodeIds: [52], + txnStream, + onFailure: () => false, + ...fields, + }); + const floors = (list) => { + const event = update({}); + event.onCommit = () => { + event.originFloors = list; + }; + return event; + }; + const ReplicatedTable = makeReplicatedTable( + 'SeqCursorLevelOneRevocationTable', + [ + { type: 'put', id: 1, value: { id: 1, name: 'first' }, timestamp: now }, + floors([ + [11, now - 20, true], + [12, now - 20, true, 'gen-1', false], + [13, now - 20, true, 'gen-1', true], + ]), + // a level-2 route now certifying at level 1 drops its generation, so a later revocation clears it + floors([[13, now - 18, true]]), + update({ originFloorRevocations: [13] }), + // a level-1 floor is cleared, not revoked, so a later certificate floors the route again + update({ originFloorRevocations: [11] }), + floors([[11, now - 15, false]]), + // coverage comes back only for a level-2 row, and only through the mark's own element + update({ originRouteStates: [[12, undefined, undefined, undefined, 'gen-1']] }), + update({ originRouteStates: [[11, undefined, undefined, undefined, undefined, true]] }), + update({ originRouteStates: [[12, undefined, undefined, undefined, 'gen-0', true]] }), + update({ originRouteStates: [[12, undefined, undefined, undefined, 'gen-1', true]] }), + { type: 'put', id: 2, value: { id: 2, name: 'second' }, timestamp: now + 1 }, + update({ localTime: now + 1, timestamp: now + 1 }), + ], + held + ); + try { + await waitFor(() => readCursor(ReplicatedTable, 52)?.seqId === now + 1, { + timeout: 5000, + message: 'the frame after the revocation and coverage updates recorded its sequence id', + }); + const nodes = readCursor(ReplicatedTable, 52).nodes.map((node) => ({ ...node })); + assert.deepEqual( + nodes.find((node) => node.id === 11), + { + id: 11, + closedFloor: now - 15, + relayable: false, + } + ); + const downgraded = nodes.find((node) => node.id === 13); + assert.equal(downgraded.closedFloor, undefined); + assert.equal(downgraded.relayable, false); + assert.equal(downgraded.originGeneration, undefined); + const levelTwo = nodes.find((node) => node.id === 12); + assert.equal(levelTwo.fullCoverage, true); + assert.equal(levelTwo.floorRevoked, undefined); + assert.equal(levelTwo.closedFloor, now - 20); + } finally { + release(); + } + }); + it('repairs a scalar whose write failed, on a repeat frame whose origin cursors did not change', async function () { let release; const held = new Promise((resolve) => (release = resolve)); diff --git a/unitTests/resources/subscribeCheckpoint.test.js b/unitTests/resources/subscribeCheckpoint.test.js index 088c6a8f3e..149a945aad 100644 --- a/unitTests/resources/subscribeCheckpoint.test.js +++ b/unitTests/resources/subscribeCheckpoint.test.js @@ -9,7 +9,9 @@ const { getThisNodeName } = require('#src/server/nodeName'); const { transaction } = require('#src/resources/transaction'); const { setOriginCaptureIntervalForTests } = require('#src/resources/transactionBroadcast'); const { + COPY_BOUND_KEY, MAX_OVERLAP_MS, + bumpOriginStateEpoch, encodeCheckpoint, membersDigest, positionBefore, @@ -504,6 +506,61 @@ describe('Per-origin subscription checkpoints (harper-pro#922 items 3–4)', fun await assert.rejects(T.subscribe({ resume: checkpoint, omitCurrent: true }), (error) => error.statusCode === 410); subscription.end(); }); + it('refuses a route a copy bound covers, certifies nothing under it, and admits once the route starts above it', async () => { + const T = tableInOwnDatabase(); + const listener = collector(); + const subscription = await T.subscribe({ omitCurrent: true, reportProgress: true, listener }); + await T.put('a', { value: 'a' }); + await waitFor(() => listener.values().includes('a')); + await sleep(2); + const floor = certify(T); + const route = { closedFloor: floor, originGeneration: 'copied-generation', fullCoverage: true }; + const peerId = writeSeqRow(T, 'copied-peer', { ...route, certifiedFrom: floor - 1000 }); + const members = membersDigest([...ownMembers(T), ['copied-peer', 'copied-generation']]); + let token; + await waitFor(() => (token = subscription.checkpoint())?.members === members); + const resumed = await T.subscribe({ resume: token, omitCurrent: true, listener: collector() }); + assert.strictEqual(await resumed.resumeVerified, true); + resumed.end(); + T.dbisDB.putSync([COPY_BOUND_KEY, 77], { [peerId]: floor - 1000 }); + bumpOriginStateEpoch(T.primaryStore.rootStore); + assert.strictEqual(subscription.checkpoint(), undefined, 'a capture from before the bound is not used'); + await assert.rejects(T.subscribe({ resume: token, omitCurrent: true }), (error) => error.statusCode === 410); + await sleep(400); + assert.strictEqual(subscription.checkpoint(), undefined, 'no capture qualifies the covered route'); + writeSeqRow(T, 'copied-peer', { ...route, certifiedFrom: floor - 999 }); + const again = await T.subscribe({ resume: token, omitCurrent: true, listener: collector() }); + assert.strictEqual(await again.resumeVerified, true); + again.end(); + await waitFor(() => subscription.checkpoint()?.members === members); + // a copy that restored this node's own lost rows: its own log is complete only above them + T.dbisDB.putSync([COPY_BOUND_KEY, 79], { 0: floor - 100_000 }); + const belowOwn = await T.subscribe({ resume: token, omitCurrent: true, listener: collector() }); + assert.strictEqual(await belowOwn.resumeVerified, true); + belowOwn.end(); + T.dbisDB.putSync([COPY_BOUND_KEY, 79], { 0: floor }); + await assert.rejects(T.subscribe({ resume: token, omitCurrent: true }), (error) => error.statusCode === 410); + bumpOriginStateEpoch(T.primaryStore.rootStore); + await sleep(400); + assert.strictEqual(subscription.checkpoint(), undefined, 'no capture while the own floor is below its start'); + T.dbisDB.removeSync([COPY_BOUND_KEY, 79]); + for (const malformed of [ + { [peerId]: 'not a version' }, + { [peerId]: -1 }, + { '': floor }, + { '-1': floor }, + { '01': floor }, + [floor], + ]) { + T.dbisDB.putSync([COPY_BOUND_KEY, 78], malformed); + await assert.rejects( + T.subscribe({ resume: token, omitCurrent: true }), + (error) => error.statusCode === 410, + `fails closed on ${JSON.stringify(malformed)}` + ); + } + subscription.end(); + }); }); describe('on LMDB', () => { diff --git a/unitTests/resources/subscriptionCheckpoint.bench.js b/unitTests/resources/subscriptionCheckpoint.bench.js new file mode 100644 index 0000000000..2ca7393987 --- /dev/null +++ b/unitTests/resources/subscriptionCheckpoint.bench.js @@ -0,0 +1,72 @@ +// The checkpoint origin-state read (`captureCertifiedFloors`, the same `readOriginStates` scan admission runs): +// 16 peer rows of 16 level-2 origins each, with BENCH_COPY_BOUND_ROWS `[copyBound, peer]` rows beside them. +const assert = require('node:assert'); +const { setupTestDBPath } = require('../testUtils'); +const { table } = require('#src/resources/databases'); +const { persistOriginClosedFloor } = require('#src/resources/auditStore'); +const { certifyOriginFloor, publishOriginFloor, getOriginClosedFloor } = require('#src/resources/originClosedFloor'); +const { getIdOfRemoteNode } = require('#src/resources/nodeIdMapping'); +const { captureCertifiedFloors } = require('#src/resources/subscriptionCheckpoint'); +const { setMainIsWorker } = require('#js/server/threads/manageThreads'); +require('#src/server/serverHelpers/serverUtilities'); + +const PEERS = 16; +const ORIGINS = 16; +const COPY_BOUND_ROWS = Number(process.env.BENCH_COPY_BOUND_ROWS ?? 0); +const ITERATIONS = 20_000; + +describe('checkpoint origin-state read', function () { + it(`captures certified floors over ${PEERS}x${ORIGINS} route entries and ${COPY_BOUND_ROWS} copy bound rows`, async function () { + this.timeout(300_000); + setupTestDBPath(); + setMainIsWorker(true); + const T = table({ + table: 'CheckpointReadBench', + database: 'checkpoint_read_bench', + audit: true, + attributes: [{ name: 'id', isPrimaryKey: true }, { name: 'value' }], + }); + await T.put('seed', { value: 0 }); + const rootStore = T.primaryStore.rootStore; + if (typeof rootStore.syncWal !== 'function') rootStore.syncWal = () => Promise.resolve(); + let floor = certifyOriginFloor(rootStore); + for (let retry = 0; floor === null && retry < 100; retry++) floor = certifyOriginFloor(rootStore); + persistOriginClosedFloor(T.auditStore, floor); + publishOriginFloor(rootStore, floor); + floor = getOriginClosedFloor(rootStore).floor; + const origins = []; + for (let o = 0; o < ORIGINS; o++) origins.push(getIdOfRemoteNode(`bench-origin-${o}`, T.auditStore)); + for (let p = 0; p < PEERS; p++) { + const peerId = getIdOfRemoteNode(`bench-peer-${p}`, T.auditStore); + T.dbisDB.putSync([Symbol.for('seq'), peerId], { + seqId: floor, + nodes: origins.map((id) => ({ + id, + originLogKey: floor, + closedFloor: floor, + relayable: true, + originGeneration: `generation-${id}`, + fullCoverage: true, + certifiedFrom: floor - 1000, + })), + }); + if (p < COPY_BOUND_ROWS) { + // below every route's start, so the routes still qualify and only the read is measured + T.dbisDB.putSync( + [Symbol.for('copyBound'), peerId], + Object.fromEntries(origins.map((id) => [id, floor - 2000])) + ); + } + } + const start = process.hrtime.bigint(); + assert.ok(captureCertifiedFloors(T.auditStore, T.dbisDB), 'every origin qualifies'); + const firstNs = Number(process.hrtime.bigint() - start); + for (let i = 0; i < 1000; i++) captureCertifiedFloors(T.auditStore, T.dbisDB); + const t0 = process.hrtime.bigint(); + for (let i = 0; i < ITERATIONS; i++) captureCertifiedFloors(T.auditStore, T.dbisDB); + const perOpNs = Number(process.hrtime.bigint() - t0) / ITERATIONS; + console.log( + `checkpoint read: copyBoundRows=${COPY_BOUND_ROWS} first=${(firstNs / 1000).toFixed(0)}us warm=${(perOpNs / 1000).toFixed(1)}us/op` + ); + }); +});