Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion core
Submodule core updated 65 files
+1 βˆ’18 bin/backup.ts
+6 βˆ’25 bin/restart.ts
+5 βˆ’10 components/mcp/listChanged.ts
+8 βˆ’60 dataLayer/DESIGN.md
+1 βˆ’5 dataLayer/backupManifest.ts
+0 βˆ’2 dataLayer/harperBridge/ResourceBridge.ts
+1 βˆ’11 dataLayer/restoreChildCommands.ts
+1 βˆ’1 dataLayer/restoreChildRunner.ts
+0 βˆ’819 dataLayer/restoreIntent.ts
+2 βˆ’3 dataLayer/restoreMarker.ts
+58 βˆ’237 dataLayer/restorePublication.ts
+5 βˆ’25 dataLayer/restoreReceipt.ts
+40 βˆ’191 dataLayer/rocksdbBackup.ts
+0 βˆ’2 dataLayer/schema.ts
+14 βˆ’111 docs/reference/backups/operations.md
+9 βˆ’10 docs/reference/backups/overview.md
+0 βˆ’6 docs/reference/configuration/overview.md
+1 βˆ’1 docs/reference/database/jobs.md
+1 βˆ’1 docs/reference/database/schema.md
+0 βˆ’2 docs/reference/http/api.md
+1 βˆ’3 docs/reference/operations-api/operations.md
+7 βˆ’0 docs/release-notes/v5-lincoln/5.4/late-message-keeps-its-record-version.md
+7 βˆ’0 docs/release-notes/v5-lincoln/5.4/replicated-messages-arrive-once-after-a-restart.md
+0 βˆ’8 docs/release-notes/v5-lincoln/5.4/restoring-a-component-held-database-at-the-next-startup.md
+0 βˆ’158 integrationTests/database/restore-on-restart.test.ts
+0 βˆ’5 integrationTests/database/restore-on-restart/config.yaml
+0 βˆ’43 integrationTests/database/restore-on-restart/resources.js
+0 βˆ’6 integrationTests/database/restore-on-restart/schema.graphql
+0 βˆ’5 integrationTests/fixtures/certificate-principal-subscriptions/config.yaml
+0 βˆ’7 integrationTests/fixtures/certificate-principal-subscriptions/resources.js
+0 βˆ’4 integrationTests/fixtures/certificate-principal-subscriptions/schema.graphql
+0 βˆ’230 integrationTests/security/certificate-principal-subscriptions.test.ts
+0 βˆ’3 json/systemSchema.json
+0 βˆ’1 package.json
+3 βˆ’1 resources/DESIGN.md
+7 βˆ’65 resources/Resource.ts
+264 βˆ’18 resources/Table.ts
+1 βˆ’5 resources/analytics/DESIGN.md
+5 βˆ’50 resources/analytics/write.ts
+0 βˆ’15 resources/databases.ts
+1 βˆ’4 resources/record-locks.md
+1 βˆ’10 security/DESIGN.md
+1 βˆ’3 security/auth.ts
+3 βˆ’109 security/user.ts
+1 βˆ’1 server/DESIGN.md
+1 βˆ’2 server/jobs/DESIGN.md
+0 βˆ’63 server/jobs/deferredJobCompletion.ts
+4 βˆ’31 server/jobs/jobOwnership.ts
+7 βˆ’20 server/jobs/jobProcess.ts
+13 βˆ’37 server/loadRootComponents.ts
+2 βˆ’3 server/mqtt.ts
+0 βˆ’43 unitTests/bin/backup.test.js
+1 βˆ’34 unitTests/components/mcp/listChanged.test.js
+0 βˆ’2,359 unitTests/dataLayer/restoreIntent.test.js
+0 βˆ’47 unitTests/dataLayer/restoreOfflineCli.test.js
+0 βˆ’131 unitTests/resources/analytics/recordAction.bench.js
+0 βˆ’139 unitTests/resources/analytics/recordAction.test.js
+278 βˆ’0 unitTests/resources/latePublishVersionRegression.test.js
+10 βˆ’4 unitTests/resources/relayedHeadTieDuplicate.test.js
+355 βˆ’0 unitTests/resources/replicatedPublishRedelivery.test.js
+1 βˆ’5 unitTests/security/auth.test.js
+0 βˆ’154 unitTests/security/userRecordLookups.test.js
+0 βˆ’176 unitTests/server/jobs/deferredJobCompletion.test.js
+0 βˆ’424 unitTests/server/liveSubscriptionAuth.test.js
+0 βˆ’1 unitTests/windowsGate.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
rest: true
graphqlSchema:
files: 'schema.graphql'
jsResource:
files: 'resources.js'
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
export class RouteState extends Resource {
static loadAsInstance = false;
post(target, data) {
const auditStore = databases.data.Message.auditStore;
const nodeLogs = auditStore.loadLogs();
const peerId = nodeLogs.indexOf(auditStore.logByName.get(data.peer));
const row = databases.data.Message.dbisDB.getSync([Symbol.for('seq'), peerId]);
const own = row?.nodes?.find((node) => node.id === peerId);
return {
seqId: row?.seqId ?? null,
closedFloor: own?.closedFloor ?? null,
originGeneration: own?.originGeneration ?? null,
fullCoverage: own?.fullCoverage ?? null,
certifiedFrom: own?.certifiedFrom ?? null,
};
}
}

export class LoggedMessages extends Resource {
static loadAsInstance = false;
post(target, data) {
const Message = databases.data.Message;
const counts = {};
Message.auditStore.loadLogs();
// getRange would create a log it does not find
if (!Message.auditStore.logByName.has(data.peer)) return counts;
for (const entry of Message.auditStore.getRange({ start: 1, log: data.peer })) {
if (entry.type !== 'message' || entry.tableId !== Message.tableId || entry.recordId !== data.topic) continue;
counts[entry.version] = (counts[entry.version] ?? 0) + 1;
}
return counts;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
type Message @table @export {
id: ID @primaryKey
n: Int
}
237 changes: 237 additions & 0 deletions integrationTests/cluster/replicatedMessageRedelivery.test.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,237 @@
/**
* A checkpoint-grade route resumes below the origin's certified floor (replication/DESIGN.md note 33), so the origin
* re-sends messages the receiver already applied. A clean-session subscriber on a restarted B can miss what B
* applied before it resubscribed, so B's own log is the oracle that holds in every case.
*/
import { suite, test, before, after } from 'node:test';
import { deepEqual, ok } from 'node:assert/strict';
import { cp, mkdtemp } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { basename, join, resolve } from 'node:path';
import { setTimeout as sleep } from 'node:timers/promises';
import mqtt from 'mqtt';
import { startHarper, getNextAvailableLoopbackAddress } from '@harperfast/integration-testing';
import {
restartHttpWorkers,
restartNode,
sendOperation,
stopAndTeardownNodes,
waitForCondition,
} from './clusterShared.mjs';

process.env.HARPER_INTEGRATION_TEST_INSTALL_SCRIPT = resolve(
import.meta.dirname,
'..',
'..',
'dist',
'bin',
'harper.js'
);

const FIXTURE = resolve(import.meta.dirname, 'fixture-replicated-messages');
const FLOOR_INTERVAL_MS = 2_000;
const PUBLISH_INTERVAL_MS = 50;
const AFTER_RESUME_MS = 15_000;
const WAIT_MS = 90_000;

const nodeOptions = (hostname) => ({
config: {
analytics: { aggregatePeriod: -1 },
logging: { colors: false, stdStreams: false, console: true },
replication: { securePort: hostname + ':9933' },
},
env: { HARPER_TEST_ORIGIN_FLOOR_INTERVAL_MS: String(FLOOR_INTERVAL_MS) },
});

async function postToFixture(node, resource, body, signal) {
const response = await fetch(`${node.httpURL}/${resource}`, {
method: 'POST',
signal: signal ?? AbortSignal.timeout(10_000),
headers: {
'Content-Type': 'application/json',
'Authorization': 'Basic ' + Buffer.from(`${node.admin.username}:${node.admin.password}`).toString('base64'),
},
body: JSON.stringify(body),
});
if (!response.ok) throw new Error(`${resource} on ${node.hostname}: ${response.status}`);
return response.json();
}

function connectOptions(node, clientId) {
return {
username: node.admin.username,
password: node.admin.password,
clientId,
clean: true,
protocolVersion: 5,
reconnectPeriod: 1000,
connectTimeout: 8000,
};
}

function startPublisher(node, topic) {
const client = mqtt.connect(`mqtt://${node.hostname}:1883`, connectOptions(node, `pub-${topic}`));
const publisher = { n: 0, client, connections: [] };
client.on('connect', () => publisher.connections.push({ at: Date.now(), nextN: publisher.n }));
const timer = setInterval(() => {
if (client.connected) client.publish(`Message/${topic}`, JSON.stringify({ n: publisher.n++ }), { qos: 0 });
}, PUBLISH_INTERVAL_MS);
publisher.stop = () => clearInterval(timer);
return publisher;
}

function subscribe(node, topic, publisher, label) {
const wsUrl = `${node.httpURL.replace(/^http/, 'ws')}/mqtt`;
const client = mqtt.connect(wsUrl, connectOptions(node, `${label}-${topic}`));
const subscriber = { client, seen: new Map(), subscriptions: [] };
client.on('connect', () =>
client.subscribe(`Message/${topic}`, { qos: 0 }, (error) => {
if (!error) subscriber.subscriptions.push({ at: Date.now(), nextN: publisher.n });
})
);
client.on('message', (_topic, payload) => {
const { n } = JSON.parse(payload.toString());
subscriber.seen.set(n, (subscriber.seen.get(n) ?? 0) + 1);
});
return subscriber;
}

const duplicates = (subscriber) => [...subscriber.seen].filter(([, count]) => count > 1).map(([n]) => n);
const missingFrom = (subscriber, from, to) =>
Array.from({ length: Math.max(to - from, 0) }, (_, i) => from + i).filter((n) => !subscriber.seen.has(n));

async function nextNAfter(events, since, description) {
return (await waitForCondition(() => events.find((event) => event.at > since), { timeoutMs: WAIT_MS, description }))
.nextN;
}

async function clusterConnected(nodes, signal) {
let statuses;
try {
statuses = await Promise.all(nodes.map((node) => sendOperation(node, { operation: 'cluster_status' }, { signal })));
} catch (error) {
// a restarted node writes its pid before it listens
if (signal.aborted) throw error;
return false;
}
return statuses.every(
(status) =>
status.connections.length === nodes.length - 1 &&
status.connections.every((connection) => connection.database_sockets.every((socket) => socket.connected))
);
}

async function deliversOnce(ctx, topic, interrupt) {
const { A, B } = ctx;
const publisher = startPublisher(A, topic);
const onB = subscribe(B, topic, publisher, 'sub-b');
const onA = subscribe(A, topic, publisher, 'sub-a');
ctx.publisher = publisher;
try {
await waitForCondition(() => onB.seen.size > 0 && onA.seen.size > 0, {
timeoutMs: WAIT_MS,
description: `live delivery of ${topic} on both nodes`,
});
const caseStart = Date.now();
let route;
await waitForCondition(
async (signal) => {
route = await postToFixture(B, 'RouteState', { peer: A.hostname }, signal);
return (
route.closedFloor > caseStart &&
route.closedFloor < route.seqId &&
typeof route.originGeneration === 'string' &&
route.fullCoverage
);
},
{
timeoutMs: WAIT_MS,
description: () => `a checkpoint-grade floor for A on B below its position, last ${JSON.stringify(route)}`,
}
);
const backUpN = await interrupt(onB);
await sleep(AFTER_RESUME_MS);
publisher.stop();
const last = publisher.n - 1;
await waitForCondition(() => onB.seen.has(last) && onA.seen.has(last), {
timeoutMs: WAIT_MS,
description: `message ${last} on both nodes`,
});
deepEqual(duplicates(onB), [], 'message numbers the subscriber on B received more than once');
deepEqual(duplicates(onA), [], 'message numbers the subscriber on A received more than once');
deepEqual(missingFrom(onB, backUpN, publisher.n), [], `numbers from ${backUpN} B never delivered`);
const logged = await postToFixture(B, 'LoggedMessages', { peer: A.hostname, topic });

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[minor] The log-count assertion can conceal messages missing during B’s restart

The log completeness check cannot detect messages dropped from the segment a clean-session subscriber legitimately missed. In the node-restart case the subscriber check starts at resubscription (integrationTests/cluster/replicatedMessageRedelivery.test.mjs:152 and integrationTests/cluster/replicatedMessageRedelivery.test.mjs:162), so messages A published while B was down are covered only by LoggedMessages, but that check counts distinct versions for the whole topic against publisher.n - backUpN; pre-interruption entries count toward the threshold, so if replay drops D of the down-time messages the distinct count is n - D and the assertion still passes whenever D <= backUpN, which already includes both the pre-interruption count and the down-time count itself. Trigger: make B's resume drop the messages committed on A during B's restart; the subscriber never sees them, no duplicate is logged, and the suite passes despite B's origin log missing that segment. Check a segment-specific oracle instead, for example the distinct versions at or after the interruption point, not a topic-wide count.

ok(
Object.keys(logged).length >= publisher.n - backUpN,
`B logged ${Object.keys(logged).length} of A's messages, fewer than the ${publisher.n - backUpN} sent after it was back`
);
deepEqual(
Object.entries(logged).filter(([, count]) => count > 1),
[],
'message versions from A that B logged more than once'
);
} finally {
publisher.stop();
for (const client of [publisher.client, onB.client, onA.client]) client.end(true);
}
}

suite('Replicated MQTT messages reach live subscribers once across resumes', { timeout: 600_000 }, (ctx) => {
before(async () => {
const hostnames = await Promise.all([0, 1].map(() => getNextAvailableLoopbackAddress()));
const roots = await Promise.all(hostnames.map(() => mkdtemp(join(tmpdir(), 'harper-integration-test-'))));
await Promise.all(
roots.map((root) => cp(FIXTURE, join(root, 'components', basename(FIXTURE)), { recursive: true }))
);
const nodes = hostnames.map((hostname, i) => ({ hostname, dataRootDir: roots[i] }));
[ctx.A, ctx.B] = nodes;
const starts = await Promise.allSettled(
nodes.map(async (node) =>
Object.assign(node, (await startHarper({ name: ctx.name, harper: node }, nodeOptions(node.hostname))).harper)
)
);
const failed = starts.find((start) => start.status === 'rejected');
if (failed) throw failed.reason;
await sendOperation(ctx.B, {
operation: 'add_node',
hostname: ctx.A.hostname,
rejectUnauthorized: false,
authorization: ctx.A.admin,
});
await waitForCondition((signal) => clusterConnected([ctx.A, ctx.B], signal), {
timeoutMs: WAIT_MS,
description: 'the cluster to connect',
});
});

after(() => stopAndTeardownNodes([ctx.A, ctx.B]));

test('the publishing node restarts while a subscriber on the other node stays connected', async () => {
await deliversOnce(ctx, 'origin-restart', async () => {
const since = Date.now();
await restartNode(ctx.A);
await waitForCondition((signal) => clusterConnected([ctx.A, ctx.B], signal), {
timeoutMs: WAIT_MS,
description: 'the cluster to reconnect',
});
return nextNAfter(ctx.publisher.connections, since, 'the publisher to reconnect to A');
});
// the lowest start a route certifies stays, so only this first resume shows it was floor-only
const { certifiedFrom } = await postToFixture(ctx.B, 'RouteState', { peer: ctx.A.hostname });
ok(certifiedFrom > 0, 'B resumed from A floor-only');
});

test("the receiving node's HTTP workers restart", () =>

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[minor] The worker-restart case does not exercise replay when replication uses the pool

With the replication pool enabled, this case does not exercise replication replay: restartHttpWorkers only rolling-restarts HTTP workers, while integrationTests/cluster/clusterShared.mjs:248-251 documents that pool-on CI legs put replication on replication.threads workers and replication/DESIGN.md:109 states those pool workers own the sockets. Trigger: run the cluster suite with HARPER_CONFIG setting replication.threads > 0, remove core's isMessageLogged drop, and this case still passes because B's A-socket never disconnects, never resumes floor-only, and never re-streams a message; only the origin-restart and node-restart cases still cover the drop. Fix by making the pool case restart the replication owner (or pin this case pool-off / branch on replicationPoolActive()), so the named scenario reaches the resume it claims to test.

deliversOnce(ctx, 'worker-restart', async (onB) => {
const since = Date.now();
await restartHttpWorkers(ctx.B);
return nextNAfter(onB.subscriptions, since, 'the subscriber on B to resubscribe');
}));

test('the receiving node restarts', () =>
deliversOnce(ctx, 'node-restart', async (onB) => {
const since = Date.now();
await restartNode(ctx.B);
return nextNAfter(onB.subscriptions, since, 'the subscriber on B to resubscribe');
}));
});
Loading
Loading