Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/glossary
Original file line number Diff line number Diff line change
Expand Up @@ -24,3 +24,4 @@ home generation: version number of the home map, which staging refuses unless ab
quiesce: post-staging drain in which a node surrenders every delegation it holds and recalls every grant it issued
lock barrier: record-lock control entry whose commit, matched on origin, position and nonce, proves that origin's prior writes applied
coordinating worker: the one worker thread per database that owns its record-lock coordination; peer subscriptions are placed independently and relay control entries to it
replication pool: the replication.threads worker threads that own every replication socket except cache-miss retrieval and operation requests a thread sends, and load no application code
69 changes: 67 additions & 2 deletions .github/workflows/integration-tests.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ jobs:
outputs:
node-versions: ${{ steps.set-node-versions.outputs.node-versions }}
build-node-versions: ${{ steps.set-node-versions.outputs.build-node-versions }}
replication-threads: ${{ steps.set-node-versions.outputs.replication-threads }}

steps:
- name: Checkout code
Expand All @@ -49,6 +50,9 @@ jobs:
# runtime for the focused cluster regression job below.
echo "node-versions=[24]" >> $GITHUB_OUTPUT
echo 'build-node-versions=[22, 24, "26.5.0"]' >> $GITHUB_OUTPUT
# The full cluster tier with the replication worker pool on runs after merge and nightly;
# PRs run the focused pool job below instead of doubling the cluster shards.
echo "replication-threads=[0]" >> $GITHUB_OUTPUT
elif [ "$NODE_VERSION_INPUT" == "all" ] || [ -z "$NODE_VERSION_INPUT" ]; then
# TEMPORARY: pinned to 26.5.0 instead of floating "26". The original reason
# (harper#2025 — an undici fetch-stall bug breaking analytics.test.mjs,
Expand All @@ -68,7 +72,9 @@ jobs:
# this pin. Do not remove it on harper#2025's closure, or the core bump, alone.
echo "node-versions=[22, 24, \"26.5.0\"]" >> $GITHUB_OUTPUT
echo 'build-node-versions=[22, 24, "26.5.0"]' >> $GITHUB_OUTPUT
echo "replication-threads=[0, 2]" >> $GITHUB_OUTPUT
else
echo "replication-threads=[0, 2]" >> $GITHUB_OUTPUT
# Manual dispatch intentionally does NOT apply the 26.5.0 pin above: selecting "26"
# here floats to the latest 26.x release, for reproducing a suspected Node-26
# regression on demand. It does not run the OCSP/CRL suites (see above) — those
Expand Down Expand Up @@ -278,8 +284,64 @@ jobs:
retention-days: 3
if-no-files-found: ignore

run-replication-pool-regression:
name: Replication Worker Pool Regression (Node.js v${{ matrix.node-version }})
if: github.event_name == 'pull_request'
runs-on: ubuntu-latest
needs: [generate-node-version-matrix, build]
timeout-minutes: 25
strategy:
fail-fast: false
matrix:
node-version: ${{ fromJSON(needs.generate-node-version-matrix.outputs.node-versions) }}

steps:
- name: Checkout code
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
submodules: 'recursive'

- name: Setup Node.js ${{ matrix.node-version }}
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
with:
node-version: ${{ matrix.node-version }}
package-manager-cache: false

- name: Download build artifacts
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: harper-build-artifacts-node-${{ matrix.node-version }}

- name: Relink bin scripts
run: npm rebuild --ignore-scripts --offline

- name: Build re2 native binary
run: .github/scripts/retry.sh "npm rebuild re2" -- npm rebuild re2

- name: Run replication suites with the worker pool on
env:
HARPER_INTEGRATION_TEST_LOG_DIR: /tmp/harper-integration-test-logs
HARPER_INTEGRATION_TEST_CONCURRENCY: '2'
HARPER_CONFIG: '{"replication":{"threads":2}}'
run: >-
npm run test:integration --
integrationTests/cluster/replicationWorkerPool.test.mjs
integrationTests/cluster/fullyConnectedReplication.test.mjs
integrationTests/cluster/recordLockCluster.test.mjs
integrationTests/cluster/cacheReplicationSource.test.mjs
integrationTests/cluster/isolatedApplicationReplication.test.mjs

- name: Upload Harper server logs
if: failure() || cancelled()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: harper-replication-pool-logs-node-${{ matrix.node-version }}
path: /tmp/harper-integration-test-logs/
retention-days: 3
if-no-files-found: ignore

run-cluster-tests:
name: Cluster Integration Tests ${{matrix.shard}}/6 (Node.js v${{ matrix.node-version }})
name: Cluster Integration Tests ${{matrix.shard}}/6 (Node.js v${{ matrix.node-version }}${{ matrix.replication-threads != 0 && ', replication pool' || '' }})
runs-on: ubuntu-latest
needs: [generate-node-version-matrix, build]
# 15 + 10: two sequential retry.sh calls (re2 rebuild, harperdb@4 install) can each
Expand All @@ -292,6 +354,7 @@ jobs:
# Heavy tier: ~40 files each booting 2–4 Harper processes — more shards than
# the non-cluster job to keep per-shard wall-clock and timeout headroom sane.
shard: [1, 2, 3, 4, 5, 6]
replication-threads: ${{ fromJSON(needs.generate-node-version-matrix.outputs.replication-threads) }}

steps:
- name: Checkout code
Expand Down Expand Up @@ -350,14 +413,16 @@ jobs:
# run heavy clusters, and the default (availableParallelism/2+1) let too
# many run at once on CI runners, starving each other into shard timeouts.
HARPER_INTEGRATION_TEST_CONCURRENCY: '2'
# Every node a suite starts inherits this; empty (pool off) is the same as unset.
HARPER_CONFIG: ${{ matrix.replication-threads != 0 && format('{{"replication":{{"threads":{0}}}}}', matrix.replication-threads) || '' }}
run: |
npm run test:integration:cluster -- --shard=${{ matrix.shard }}/6

- name: Upload Harper server logs
if: failure()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: harper-cluster-server-logs-node-${{ matrix.node-version }}-shard-${{ matrix.shard }}
name: harper-cluster-server-logs-node-${{ matrix.node-version }}-shard-${{ matrix.shard }}-replication-threads-${{ matrix.replication-threads }}
path: /tmp/harper-integration-test-logs/
retention-days: 3
if-no-files-found: ignore
5 changes: 5 additions & 0 deletions integrationTests/cluster/fixture-computed-index/config.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
graphqlSchema:
files: schema.graphql
jsResource:
files: resources.js
rest: true
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
import { tables } from 'harper';

tables.ComputedRow.setComputedAttribute('upper', (record) => record.name?.toUpperCase());
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
type ComputedRow @table @export {
id: ID @primaryKey
name: String
upper: String @computed @indexed
shout: String @computed(from: "name + '!'") @indexed
}
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,10 @@ async function threadsOf(node) {
const dedicated = threads.filter((thread) => thread.application === ISOLATED_APP).map((t) => t.threadId);
equal(dedicated.length, 1, `expected one dedicated worker: ${JSON.stringify(threads)}`);
equal(pool.length, 1, `expected one pool worker: ${JSON.stringify(threads)}`);
return { pool, dedicated: dedicated[0] };
// With replication.threads (the pool-on CI jobs set it through HARPER_CONFIG), the replication pool owns
// subscriptions and record-lock coordination instead of the http workers.
const replicationPool = threads.filter((thread) => thread.name === 'replication').map((t) => t.threadId);
return { pool: replicationPool.length > 0 ? replicationPool : pool, dedicated: dedicated[0] };
}

suite('replication stays off an isolated application worker', { skip: UNSUPPORTED_HERE, timeout: 240_000 }, (ctx) => {
Expand Down
21 changes: 20 additions & 1 deletion integrationTests/cluster/recordLockShared.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,20 @@ process.env.HARPER_INTEGRATION_TEST_INSTALL_SCRIPT = join(import.meta.dirname, '
const FIXTURE = join(import.meta.dirname, 'fixture-record-locks');
export const DB = 'data';
export const CONVERGE_TIMEOUT_MS = 90_000;
/**
* With the replication pool on (the pool-on CI jobs set `replication.threads` for every node through
* `HARPER_CONFIG`), the pool owns subscriptions and record-lock coordination, so a multi-worker node gets as
* many pool workers as http workers. At 2 pool workers, round-robin over (peer, database) with two databases
* would put every peer's `data` stream on one worker, never off the coordinating one.
*/
function poolEnv(threadsCount) {
const shared = process.env.HARPER_CONFIG ? JSON.parse(process.env.HARPER_CONFIG) : undefined;
if (threadsCount <= 1 || !(shared?.replication?.threads > 0)) return {};
return {
HARPER_CONFIG: JSON.stringify({ ...shared, replication: { ...shared.replication, threads: threadsCount } }),
};
}

function optionsFor(hostname, env = {}, threadsCount = 1) {
return {
config: {
Expand All @@ -29,7 +43,12 @@ function optionsFor(hostname, env = {}, threadsCount = 1) {
pingTimeout: 3000,
},
},
env: { HARPER_NO_FLUSH_ON_EXIT: true, HARPER_TEST_RECORD_LOCK_MIN_DRAIN_BACKSTOP_MS: '0', ...env },
env: {
HARPER_NO_FLUSH_ON_EXIT: true,
HARPER_TEST_RECORD_LOCK_MIN_DRAIN_BACKSTOP_MS: '0',
...poolEnv(threadsCount),
...env,
},
};
}

Expand Down
188 changes: 188 additions & 0 deletions integrationTests/cluster/replicationWorkerPool.test.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,188 @@
/**
* With `replication.threads`, every replication socket (the replication port's inbound listener and
* every outbound subscription) is owned by a `replication` pool worker, never an HTTP worker.
*/
import { suite, test, before, after } from 'node:test';
import { ok, strictEqual } from 'node:assert';
import { startHarper, teardownHarper, getNextAvailableLoopbackAddress } from '@harperfast/integration-testing';
import { join } from 'node:path';
import { sendOperation, readLog, waitForCondition } from './clusterShared.mjs';

process.env.HARPER_INTEGRATION_TEST_INSTALL_SCRIPT = join(import.meta.dirname, '..', '..', 'dist', 'bin', 'harper.js');

const NODE_COUNT = 2;
const POOL_SIZE = 2;

async function threadIds(node, name, signal) {
const { threads } = await sendOperation(
node,
{ operation: 'system_information', attributes: ['threads'] },
{ signal }
);
return threads.filter((thread) => thread.name === name).map((thread) => thread.threadId);
}

async function outboundSocketThreads(node, signal) {
const { connections } = await sendOperation(node, { operation: 'cluster_status' }, { signal });
return connections.flatMap((connection) =>
connection.database_sockets.map((socket) => ({ ...socket, peer: connection.name }))
);
}

async function allConnected(nodes, signal) {
for (const node of nodes) {
const sockets = await outboundSocketThreads(node, signal);
if (sockets.length === 0 || !sockets.every((socket) => socket.connected)) return false;
}
return true;
}

async function readReplicated(node, table, id, signal) {
const [record] = await sendOperation(
node,
{
operation: 'search_by_id',
table,
get_attributes: ['id', 'name'],
ids: [id],
},
{ signal }
);
return record;
}

/** The thread type of every log line recording an accepted inbound replication socket. */
async function inboundSocketThreadTypes(node) {
const log = await readLog(node);
return log
.split('\n')
.filter((line) => line.includes('Incoming replication WS connection received'))
.map((line) => line.match(/ \[([a-z]+)\/\d+\]/)?.[1]);
}

suite('replication runs on the dedicated worker pool', (ctx) => {
before(async () => {
const started = await Promise.allSettled(
Array.from({ length: NODE_COUNT }, async () => {
const nodeCtx = { name: ctx.name, harper: { hostname: await getNextAvailableLoopbackAddress() } };
try {
await startHarper(nodeCtx, {
config: {
analytics: { aggregatePeriod: -1 },
logging: { colors: false, stdStreams: false, console: true, level: 'debug' },
replication: { securePort: nodeCtx.harper.hostname + ':9933', threads: POOL_SIZE },
},
env: { HARPER_NO_FLUSH_ON_EXIT: true },
});
} catch (error) {
// not yet in ctx.nodes, so release the reserved address (and any data root) here
await teardownHarper(nodeCtx).catch(() => {});
throw error;
}
return nodeCtx.harper;
})
);
// a node that did start must still reach after()'s teardown when its sibling failed
ctx.nodes = started.filter((result) => result.status === 'fulfilled').map((result) => result.value);
const failed = started.find((result) => result.status === 'rejected');
if (failed) throw failed.reason;
for (const node of ctx.nodes) {
await sendOperation(node, {
operation: 'create_table',
table: 'pooled',
primary_key: 'id',
attributes: [
{ name: 'id', type: 'ID' },
{ name: 'name', type: 'String' },
],
});
}
const { operation_token: token } = await sendOperation(ctx.nodes[0], {
operation: 'create_authentication_tokens',
authorization: ctx.nodes[0].admin,
});
await sendOperation(ctx.nodes[1], {
operation: 'add_node',
rejectUnauthorized: false,
hostname: ctx.nodes[0].hostname,
authorization: 'Bearer ' + token,
});
await waitForCondition((signal) => allConnected(ctx.nodes, signal), { description: 'the cluster to connect' });
});

after(async () => {
if (ctx.nodes) await Promise.all(ctx.nodes.map((node) => teardownHarper({ harper: node })));
});

test('starts the pool beside the HTTP workers', async () => {
for (const node of ctx.nodes) strictEqual((await threadIds(node, 'replication')).length, POOL_SIZE);
});

test('every outbound subscription is owned by a pool worker', async () => {
for (const node of ctx.nodes) {
const pool = await threadIds(node, 'replication');
const sockets = await outboundSocketThreads(node);
ok(sockets.length > 0, 'the node subscribes to its peer');
for (const socket of sockets)
ok(
pool.includes(socket.threadId),
`${socket.peer}/${socket.database} is on thread ${socket.threadId}, not the pool ${pool}`
);
}
});

test('every inbound replication socket is accepted by a pool worker', async () => {
for (const node of ctx.nodes) {
const types = await inboundSocketThreadTypes(node);
ok(types.length > 0, 'the node accepted its peer’s subscriptions');
ok(
types.every((type) => type === 'replication'),
`inbound replication sockets were accepted on: ${types}`
);
}
});

test('replicates writes, including to a table created after the pool started', async () => {
await sendOperation(ctx.nodes[0], {
operation: 'upsert',
table: 'pooled',
records: [{ id: '1', name: 'first' }],
replicatedConfirmation: 1,
});
const record = await waitForCondition((signal) => readReplicated(ctx.nodes[1], 'pooled', '1', signal), {
timeoutMs: 30000,
description: 'the write to replicate',
});
strictEqual(record.name, 'first');
});

test('reconnects on a replaced pool after an operator restart', async () => {
const node = ctx.nodes[1];
const before = await threadIds(node, 'replication');
await sendOperation(node, { operation: 'restart_service', service: 'http_workers' });
const replaced = await waitForCondition(
async (signal) => {
const pool = await threadIds(node, 'replication', signal);
return pool.length === POOL_SIZE && pool.every((id) => !before.includes(id)) && pool;
},
{ timeoutMs: 60000, description: 'the pool to be replaced' }
);
await waitForCondition(
async (signal) => {
const sockets = await outboundSocketThreads(node, signal);
return sockets.length > 0 && sockets.every((socket) => socket.connected && replaced.includes(socket.threadId));
},
{ timeoutMs: 60000, description: 'subscriptions to move to the new pool and reconnect' }
);
await sendOperation(ctx.nodes[0], {
operation: 'upsert',
table: 'pooled',
records: [{ id: '2', name: 'after restart' }],
});
const record = await waitForCondition((signal) => readReplicated(node, 'pooled', '2', signal), {
timeoutMs: 30000,
description: 'a write after the restart to replicate',
});
strictEqual(record.name, 'after restart');
});
});
Loading
Loading