diff --git a/core b/core index fe3b268e4..6a0669337 160000 --- a/core +++ b/core @@ -1 +1 @@ -Subproject commit fe3b268e4be92658d7509a4e21f020632c8efc79 +Subproject commit 6a066933752eb7fd0d3166588efc56cf5dbd15ba diff --git a/integrationTests/cluster/fixture-replicated-messages/config.yaml b/integrationTests/cluster/fixture-replicated-messages/config.yaml new file mode 100644 index 000000000..41005f90a --- /dev/null +++ b/integrationTests/cluster/fixture-replicated-messages/config.yaml @@ -0,0 +1,5 @@ +rest: true +graphqlSchema: + files: 'schema.graphql' +jsResource: + files: 'resources.js' diff --git a/integrationTests/cluster/fixture-replicated-messages/resources.js b/integrationTests/cluster/fixture-replicated-messages/resources.js new file mode 100644 index 000000000..ff2c83d7f --- /dev/null +++ b/integrationTests/cluster/fixture-replicated-messages/resources.js @@ -0,0 +1,33 @@ +export class RouteState extends Resource { + static loadAsInstance = false; + post(target, data) { + const auditStore = databases.data.Message.auditStore; + const nodeLogs = auditStore.loadLogs(); + const peerId = nodeLogs.indexOf(auditStore.logByName.get(data.peer)); + const row = databases.data.Message.dbisDB.getSync([Symbol.for('seq'), peerId]); + const own = row?.nodes?.find((node) => node.id === peerId); + return { + seqId: row?.seqId ?? null, + closedFloor: own?.closedFloor ?? null, + originGeneration: own?.originGeneration ?? null, + fullCoverage: own?.fullCoverage ?? null, + certifiedFrom: own?.certifiedFrom ?? null, + }; + } +} + +export class LoggedMessages extends Resource { + static loadAsInstance = false; + post(target, data) { + const Message = databases.data.Message; + const counts = {}; + Message.auditStore.loadLogs(); + // getRange would create a log it does not find + if (!Message.auditStore.logByName.has(data.peer)) return counts; + for (const entry of Message.auditStore.getRange({ start: 1, log: data.peer })) { + if (entry.type !== 'message' || entry.tableId !== Message.tableId || entry.recordId !== data.topic) continue; + counts[entry.version] = (counts[entry.version] ?? 0) + 1; + } + return counts; + } +} diff --git a/integrationTests/cluster/fixture-replicated-messages/schema.graphql b/integrationTests/cluster/fixture-replicated-messages/schema.graphql new file mode 100644 index 000000000..f875b61f7 --- /dev/null +++ b/integrationTests/cluster/fixture-replicated-messages/schema.graphql @@ -0,0 +1,4 @@ +type Message @table @export { + id: ID @primaryKey + n: Int +} diff --git a/integrationTests/cluster/replicatedMessageRedelivery.test.mjs b/integrationTests/cluster/replicatedMessageRedelivery.test.mjs new file mode 100644 index 000000000..01c854300 --- /dev/null +++ b/integrationTests/cluster/replicatedMessageRedelivery.test.mjs @@ -0,0 +1,237 @@ +/** + * A checkpoint-grade route resumes below the origin's certified floor (replication/DESIGN.md note 33), so the origin + * re-sends messages the receiver already applied. A clean-session subscriber on a restarted B can miss what B + * applied before it resubscribed, so B's own log is the oracle that holds in every case. + */ +import { suite, test, before, after } from 'node:test'; +import { deepEqual, ok } from 'node:assert/strict'; +import { cp, mkdtemp } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { basename, join, resolve } from 'node:path'; +import { setTimeout as sleep } from 'node:timers/promises'; +import mqtt from 'mqtt'; +import { startHarper, getNextAvailableLoopbackAddress } from '@harperfast/integration-testing'; +import { + restartHttpWorkers, + restartNode, + sendOperation, + stopAndTeardownNodes, + waitForCondition, +} from './clusterShared.mjs'; + +process.env.HARPER_INTEGRATION_TEST_INSTALL_SCRIPT = resolve( + import.meta.dirname, + '..', + '..', + 'dist', + 'bin', + 'harper.js' +); + +const FIXTURE = resolve(import.meta.dirname, 'fixture-replicated-messages'); +const FLOOR_INTERVAL_MS = 2_000; +const PUBLISH_INTERVAL_MS = 50; +const AFTER_RESUME_MS = 15_000; +const WAIT_MS = 90_000; + +const nodeOptions = (hostname) => ({ + config: { + analytics: { aggregatePeriod: -1 }, + logging: { colors: false, stdStreams: false, console: true }, + replication: { securePort: hostname + ':9933' }, + }, + env: { HARPER_TEST_ORIGIN_FLOOR_INTERVAL_MS: String(FLOOR_INTERVAL_MS) }, +}); + +async function postToFixture(node, resource, body, signal) { + const response = await fetch(`${node.httpURL}/${resource}`, { + method: 'POST', + signal: signal ?? AbortSignal.timeout(10_000), + headers: { + 'Content-Type': 'application/json', + 'Authorization': 'Basic ' + Buffer.from(`${node.admin.username}:${node.admin.password}`).toString('base64'), + }, + body: JSON.stringify(body), + }); + if (!response.ok) throw new Error(`${resource} on ${node.hostname}: ${response.status}`); + return response.json(); +} + +function connectOptions(node, clientId) { + return { + username: node.admin.username, + password: node.admin.password, + clientId, + clean: true, + protocolVersion: 5, + reconnectPeriod: 1000, + connectTimeout: 8000, + }; +} + +function startPublisher(node, topic) { + const client = mqtt.connect(`mqtt://${node.hostname}:1883`, connectOptions(node, `pub-${topic}`)); + const publisher = { n: 0, client, connections: [] }; + client.on('connect', () => publisher.connections.push({ at: Date.now(), nextN: publisher.n })); + const timer = setInterval(() => { + if (client.connected) client.publish(`Message/${topic}`, JSON.stringify({ n: publisher.n++ }), { qos: 0 }); + }, PUBLISH_INTERVAL_MS); + publisher.stop = () => clearInterval(timer); + return publisher; +} + +function subscribe(node, topic, publisher, label) { + const wsUrl = `${node.httpURL.replace(/^http/, 'ws')}/mqtt`; + const client = mqtt.connect(wsUrl, connectOptions(node, `${label}-${topic}`)); + const subscriber = { client, seen: new Map(), subscriptions: [] }; + client.on('connect', () => + client.subscribe(`Message/${topic}`, { qos: 0 }, (error) => { + if (!error) subscriber.subscriptions.push({ at: Date.now(), nextN: publisher.n }); + }) + ); + client.on('message', (_topic, payload) => { + const { n } = JSON.parse(payload.toString()); + subscriber.seen.set(n, (subscriber.seen.get(n) ?? 0) + 1); + }); + return subscriber; +} + +const duplicates = (subscriber) => [...subscriber.seen].filter(([, count]) => count > 1).map(([n]) => n); +const missingFrom = (subscriber, from, to) => + Array.from({ length: Math.max(to - from, 0) }, (_, i) => from + i).filter((n) => !subscriber.seen.has(n)); + +async function nextNAfter(events, since, description) { + return (await waitForCondition(() => events.find((event) => event.at > since), { timeoutMs: WAIT_MS, description })) + .nextN; +} + +async function clusterConnected(nodes, signal) { + let statuses; + try { + statuses = await Promise.all(nodes.map((node) => sendOperation(node, { operation: 'cluster_status' }, { signal }))); + } catch (error) { + // a restarted node writes its pid before it listens + if (signal.aborted) throw error; + return false; + } + return statuses.every( + (status) => + status.connections.length === nodes.length - 1 && + status.connections.every((connection) => connection.database_sockets.every((socket) => socket.connected)) + ); +} + +async function deliversOnce(ctx, topic, interrupt) { + const { A, B } = ctx; + const publisher = startPublisher(A, topic); + const onB = subscribe(B, topic, publisher, 'sub-b'); + const onA = subscribe(A, topic, publisher, 'sub-a'); + ctx.publisher = publisher; + try { + await waitForCondition(() => onB.seen.size > 0 && onA.seen.size > 0, { + timeoutMs: WAIT_MS, + description: `live delivery of ${topic} on both nodes`, + }); + const caseStart = Date.now(); + let route; + await waitForCondition( + async (signal) => { + route = await postToFixture(B, 'RouteState', { peer: A.hostname }, signal); + return ( + route.closedFloor > caseStart && + route.closedFloor < route.seqId && + typeof route.originGeneration === 'string' && + route.fullCoverage + ); + }, + { + timeoutMs: WAIT_MS, + description: () => `a checkpoint-grade floor for A on B below its position, last ${JSON.stringify(route)}`, + } + ); + const backUpN = await interrupt(onB); + await sleep(AFTER_RESUME_MS); + publisher.stop(); + const last = publisher.n - 1; + await waitForCondition(() => onB.seen.has(last) && onA.seen.has(last), { + timeoutMs: WAIT_MS, + description: `message ${last} on both nodes`, + }); + deepEqual(duplicates(onB), [], 'message numbers the subscriber on B received more than once'); + deepEqual(duplicates(onA), [], 'message numbers the subscriber on A received more than once'); + deepEqual(missingFrom(onB, backUpN, publisher.n), [], `numbers from ${backUpN} B never delivered`); + const logged = await postToFixture(B, 'LoggedMessages', { peer: A.hostname, topic }); + ok( + Object.keys(logged).length >= publisher.n - backUpN, + `B logged ${Object.keys(logged).length} of A's messages, fewer than the ${publisher.n - backUpN} sent after it was back` + ); + deepEqual( + Object.entries(logged).filter(([, count]) => count > 1), + [], + 'message versions from A that B logged more than once' + ); + } finally { + publisher.stop(); + for (const client of [publisher.client, onB.client, onA.client]) client.end(true); + } +} + +suite('Replicated MQTT messages reach live subscribers once across resumes', { timeout: 600_000 }, (ctx) => { + before(async () => { + const hostnames = await Promise.all([0, 1].map(() => getNextAvailableLoopbackAddress())); + const roots = await Promise.all(hostnames.map(() => mkdtemp(join(tmpdir(), 'harper-integration-test-')))); + await Promise.all( + roots.map((root) => cp(FIXTURE, join(root, 'components', basename(FIXTURE)), { recursive: true })) + ); + const nodes = hostnames.map((hostname, i) => ({ hostname, dataRootDir: roots[i] })); + [ctx.A, ctx.B] = nodes; + const starts = await Promise.allSettled( + nodes.map(async (node) => + Object.assign(node, (await startHarper({ name: ctx.name, harper: node }, nodeOptions(node.hostname))).harper) + ) + ); + const failed = starts.find((start) => start.status === 'rejected'); + if (failed) throw failed.reason; + await sendOperation(ctx.B, { + operation: 'add_node', + hostname: ctx.A.hostname, + rejectUnauthorized: false, + authorization: ctx.A.admin, + }); + await waitForCondition((signal) => clusterConnected([ctx.A, ctx.B], signal), { + timeoutMs: WAIT_MS, + description: 'the cluster to connect', + }); + }); + + after(() => stopAndTeardownNodes([ctx.A, ctx.B])); + + test('the publishing node restarts while a subscriber on the other node stays connected', async () => { + await deliversOnce(ctx, 'origin-restart', async () => { + const since = Date.now(); + await restartNode(ctx.A); + await waitForCondition((signal) => clusterConnected([ctx.A, ctx.B], signal), { + timeoutMs: WAIT_MS, + description: 'the cluster to reconnect', + }); + return nextNAfter(ctx.publisher.connections, since, 'the publisher to reconnect to A'); + }); + // the lowest start a route certifies stays, so only this first resume shows it was floor-only + const { certifiedFrom } = await postToFixture(ctx.B, 'RouteState', { peer: ctx.A.hostname }); + ok(certifiedFrom > 0, 'B resumed from A floor-only'); + }); + + test("the receiving node's HTTP workers restart", () => + deliversOnce(ctx, 'worker-restart', async (onB) => { + const since = Date.now(); + await restartHttpWorkers(ctx.B); + return nextNAfter(onB.subscriptions, since, 'the subscriber on B to resubscribe'); + })); + + test('the receiving node restarts', () => + deliversOnce(ctx, 'node-restart', async (onB) => { + const since = Date.now(); + await restartNode(ctx.B); + return nextNAfter(onB.subscriptions, since, 'the subscriber on B to resubscribe'); + })); +}); diff --git a/replication/DESIGN.md b/replication/DESIGN.md index f2b81b422..a590eb958 100644 --- a/replication/DESIGN.md +++ b/replication/DESIGN.md @@ -596,7 +596,7 @@ Regressions: `unitTests/replication/replicateFalseSendPaths.test.mjs` (frames), 33. **Checkpoint-grade certificates, floor-only resume and revocation (harper-pro#922 items 3–4; the consumer is core's "Per-origin subscription checkpoints").** `originFloors` level 2 is advertised only by a RocksDB build whose database instance syncs the floor record before it is advertised (`originFloorsDurable`), and resolved as `min(local, peer)`. - **Wire.** `[SEQUENCE_ID_UPDATE, position, { origin: floor | [floor, generation, fullCoverage] | 0 }]` (`parseOriginCertificates`, `encodeOriginCertificate`): a level-1 peer still gets `{ origin: floor }` and no `0`. The sender's own certificate carries its database generation with `fullCoverage: true`; a relayed one carries the level-2 fields only from a row whose own socket certifies at level 2 now (`PEER_CERTIFIES_FLOORS_POSITION` holds the level, 0/1/2), and a row that still holds a generation on a socket that dropped a level relays nothing until its revocation (a bare floor sent first would clear the generation downstream, and the revocation would then find none to revoke within) and whose `certifiedFrom` is a recorded number (`0` or at or below the subscriber's requested start for that origin, falling back to `currentSequenceId` when the subscriber sent no per-origin cursor; `collectRelayableCertificates`), and a revoked row relays `0` once per connection and last known generation (cleared if a newer valid certificate is later sent). - **Receiver** (`receivedOriginFloors`): on a level-2 socket a tuple is stored as `[id, floor, relayable, generation, fullCoverage]` where `fullCoverage` is true only when this socket replicates every table to this node (`exclusionOrigins` names the peer) and, for a relayed origin, the sender's route did too; `0` becomes `originFloorRevocations` on the same sequence update (naming no generation for a genless row, which core clears rather than revokes), and while a revocation is pending for an origin on that socket (`hasPendingRevocation`), incoming certificates for that origin are ignored so a concurrent floor cannot clear `floorRevoked` before the revocation commits. Core merges the qualifiers into `nodes[]` (a non-advancing `[seq, peer]` write still merges qualifiers, route marks, and revocations) and revokes — sticky within that origin generation, floor cleared; operator recovery from a rolled-back/downgraded origin requires re-cloning or rebuilding that origin so its database generation rotates — on a revocation, or when a certificate names another origin generation. `PendingOriginFloors.takeForCommit` restores taken floors if the carrying transaction fails (`onFailure`), updates `fullCoverage` when a same-floor certificate under the same generation changes coverage, and retains the last seen generation per origin across takes so a re-created origin's lower floor replaces an unstamped pending entry. - - **Resume is floor-only on a checkpoint-grade row** (`cursorBelowFloor(F)`, never `max(position, …)`, and only once `cursorBelowFloor(F)` has advanced past the highest audit-less `copyApply` record version applied for that origin across the database, persisted per copying peer in `[Symbol.for('copyBound'), peerId]` and merged across all peers): a transaction reserved below the sender's position and committed during the disconnect is re-sent instead of filtered, which is what lets a stored floor mean a complete local prefix, while the leading-duplicate fast-skip arms from `max(seqId, startTime)` on direct routes and `max(seqId, originLogKey)` on relayed routes so the re-streamed overlap past the floor is elided without driving core's resequencing walk. The floor-only start is recorded as the route's `certifiedFrom` (`PendingRouteState`, lowest wins; `originRouteStates` on the next sequence update; only from a socket's first request, see **Certification needs a fresh socket**; for a relayed origin, `currentOriginCursorVector` holds the floor-only start in `optimisticCertifiedOrigins` and promotes it to `PendingRouteState` only once `receivedOriginFloors` confirms a level-2 certificate with `fullCoverage: true`, while a bare or partial-coverage floor clears any optimistic or stored `certifiedFrom`), a `certifiedFrom` at or below the origin's database-wide copy bound is void (see **Row ownership** below), a dropped record marks the origin `hole` and flushes immediately, a level-2 route widened to full coverage or narrowed from full coverage marks `partialHistory` until a full base copy completes (checking the direct peer's own row on `[seq, peer]` and `copyCursor.fullCoverage` so an unrelated relayed origin's partial coverage or a restarted full-coverage copy does not force another full copy restart, restarting any interrupted partial-coverage copy from the first table via `copyCursor.restartFullCopy`, and automatically requesting a full base copy (`startTime = 0`) when coverage widens without an active copy to populate newly covered tables), and a direct peer whose handshake no longer certifies at level 2 while its row holds a generation is revoked immediately in both `PendingRouteState` and the in-memory `seq` snapshot used to build the resume request. + - **Resume is floor-only on a checkpoint-grade row** (`cursorBelowFloor(F)`, never `max(position, …)`, and only once `cursorBelowFloor(F)` has advanced past the highest audit-less `copyApply` record version applied for that origin across the database, persisted per copying peer in `[Symbol.for('copyBound'), peerId]` and merged across all peers): a transaction reserved below the sender's position and committed during the disconnect is re-sent instead of filtered, which is what lets a stored floor mean a complete local prefix, while the leading-duplicate fast-skip arms from `max(seqId, startTime)` on direct routes and `max(seqId, originLogKey)` on relayed routes so the re-streamed overlap past the floor is elided without driving core's resequencing walk; that skip ties only a stored record, so a re-streamed publish (a message, which stores none) is dropped by core's apply, which finds its entry already in the origin's log (`isMessageLogged`), or every live subscriber would receive it again. The floor-only start is recorded as the route's `certifiedFrom` (`PendingRouteState`, lowest wins; `originRouteStates` on the next sequence update; only from a socket's first request, see **Certification needs a fresh socket**; for a relayed origin, `currentOriginCursorVector` holds the floor-only start in `optimisticCertifiedOrigins` and promotes it to `PendingRouteState` only once `receivedOriginFloors` confirms a level-2 certificate with `fullCoverage: true`, while a bare or partial-coverage floor clears any optimistic or stored `certifiedFrom`), a `certifiedFrom` at or below the origin's database-wide copy bound is void (see **Row ownership** below), a dropped record marks the origin `hole` and flushes immediately, a level-2 route widened to full coverage or narrowed from full coverage marks `partialHistory` until a full base copy completes (checking the direct peer's own row on `[seq, peer]` and `copyCursor.fullCoverage` so an unrelated relayed origin's partial coverage or a restarted full-coverage copy does not force another full copy restart, restarting any interrupted partial-coverage copy from the first table via `copyCursor.restartFullCopy`, and automatically requesting a full base copy (`startTime = 0`) when coverage widens without an active copy to populate newly covered tables), and a direct peer whose handshake no longer certifies at level 2 while its row holds a generation is revoked immediately in both `PendingRouteState` and the in-memory `seq` snapshot used to build the resume request. - Marks and revocations name the origin generation they were derived under (core ignores one whose generation is no longer the row's), ride the next sequence update of any shape (streamed transaction `endTxnEvent`, sequence heartbeat, or `flushRouteState`), and settle when that event's `onCommit` runs (`updateRecordedSequenceId` merges route marks and revocations even when `seqId` does not advance, `PendingRouteState` restores settled safety marks and revocations if `onFailure` runs before `updateRecordedSequenceId` or if a post-`updateRecordedSequenceId` barrier `end_txn` verification read of `[seq, peer]` finds that `updateRecordedSequenceId` did not persist them, and events with an empty `remoteNodeIds` list are skipped so marks are not settled without a persisted `[seq, peer]` write; `SharedRouteSafetyState` retains `PendingRouteState` safety marks, `partialMarked`, `holedOrigins`, and `revokedOrigins` per `(database, route node)` within a worker across socket replacement while stripping unverified positive certification on reconnect, re-asserting any mark or revocation whose `[seq, peer]` write did not land — for example when core's `updateRecordedSequenceId` rejected without calling `onFailure` — once `[seq, peer]` holds a generation, and dropping the entry if a newer generation rotates in). `PendingRouteState` coalesces marks per origin (`clearCertifiedFirst` emits an intermediate `partialHistory: true` clear before `partialHistory: false` when both land in the same batch, and `certifiedFrom` is ignored while `partialHistory: true` is pending), `CopyPassState` (`restoreCompletedCopyPass`) restores `partialHistory: false` with each origin's captured generation across every full-coverage base-copy finalization path (leaving `partialHistory: true` intact when `fullCoverage === false` or `resumedWithWidening === true`) and awaits `[Symbol.for('copyBound'), peerId]` persistence and the route's `fullCoverage: true` (restored by a route mark's sixth element) on `[seq, peer]` before removing `[copyCursor, peer]` — queueing a post-`updateRecordedSequenceId` barrier `end_txn` event on `txnStream` when restoration marks ride the finalization event so `[copyCursor, peer]` is removed only after `[seq, peer]` durably reflects `partialHistory: false` (re-queueing `partialHistory: false` and deferring cursor removal via `deferCopyCursorRemoval` with a 1 s retry timer if `persistCopyBounds()`, durable route verification, or `remove` fails; base copies never set `certifiedFrom`, because `copyApply` snapshot rows write no local audit entries; `certifiedFrom` is earned on the first floor-only resume once `closedFloor` advances past the highest copied record version), `copyCursor.widened` and `copyCursor.fullCoverage` persist coverage state across an interrupted copy, and `COPY_START` marks all known origins on the peer's row as `partialHistory: true` until the copy finishes. `flushRouteState` sends a sequence update of this node's own right after a subscription request (never inside an open frame, never awaiting the subscription placeholder, and closing with 1011 if the placeholder never resolves after 60 retries), since a downgraded peer sends none; a revocation is accepted during a copy (a copy's floors are not) and drops that origin's held certificate. A downgrade, a widening, or a narrowing marks every origin on that peer's row, relayed ones included; a relay never forwards the level-2 fields from a row with a hole, partial history, an unrecorded `certifiedFrom`, `certifiedFrom` at or below the origin's `copyBound`, or `certifiedFrom` above the subscriber's requested start, nor a level-2 tuple once the origin's socket dropped a level; a floor-only start is never raised by a recorded transaction time, which would reopen the gap; a widening restarts an interrupted partial-coverage copy from the first table and only a full copy begun after the mark clears it; a copy's request records no `certifiedFrom`. - **The route's row.** A connection's route state lives on `[seq, remoteNodeIds[0]]`, the row core's updater writes for its sequence updates: the first subscribed node (`receivingDataFromNodeIds[0]`), which on a failover subscription (another node's data through this socket's peer) is not the peer. Marks, the post-commit read-back (each batch against the row its own event wrote, all rows read before any batch is consumed), copy-pass restoration (`CopyPassState.routeId`) and `SharedRouteSafetyState` (keyed by that node; a socket with no subscription binds none) all use it; the copy cursor stays keyed by the copy's source. A read-back that fails retries on the 250 ms timer, never inline: one that could never pass, reading the socket peer's row, once held a worker at 100% and with it every sharding bootstrap RPC (`shardingBootstrap.test.mjs`). - **Certification needs a fresh socket.** Only a socket's first subscription request records `certifiedFrom` or optimistic starts. A later request, or a `SUBSCRIPTION_UPDATE` include, on the same socket keeps its floor-only starts (the row's existing `certifiedFrom` needs them for continuity) but certifies nothing new: until the sender acts on it, its certificates come from the old scan, which never re-sent what the new start covers. A sender captures floors before a scan and sends them after it drains, so on a socket with one subscription every certificate post-dates that subscription's replay.