From f070c4a488650556c06e31dc3ad8fb05817fece4 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Mon, 26 Oct 2026 06:54:11 -0600 Subject: [PATCH 1/4] test: characterize pooled static-sharding limit queries Dispatch-Task: harper-pro-main-red-static-sharding-20261022 Co-Authored-By: GPT-5 Codex --- .github/workflows/integration-tests.yaml | 1 + .../fixture-static-sharding/resources.js | 30 +++++++++++- .../cluster/staticSharding.test.mjs | 46 +++++++++++++++++-- replication/DESIGN.md | 2 +- 4 files changed, 73 insertions(+), 6 deletions(-) diff --git a/.github/workflows/integration-tests.yaml b/.github/workflows/integration-tests.yaml index d440f3ba2..eceb48fd9 100644 --- a/.github/workflows/integration-tests.yaml +++ b/.github/workflows/integration-tests.yaml @@ -476,6 +476,7 @@ jobs: integrationTests/cluster/recordLockCluster.test.mjs integrationTests/cluster/cacheReplicationSource.test.mjs integrationTests/cluster/isolatedApplicationReplication.test.mjs + integrationTests/cluster/staticSharding.test.mjs - name: Upload Harper server logs if: failure() || cancelled() diff --git a/integrationTests/cluster/fixture-static-sharding/resources.js b/integrationTests/cluster/fixture-static-sharding/resources.js index e76eb4d00..76d16e2d8 100644 --- a/integrationTests/cluster/fixture-static-sharding/resources.js +++ b/integrationTests/cluster/fixture-static-sharding/resources.js @@ -2,9 +2,37 @@ for (const table of [tables.Sharded, tables.OtherSharded]) table.setResidencyById((id) => Number(id.match(/^(\d+):/)?.[1] ?? 0)); tables.Placed.setResidency((record) => record.homes); +// GET_RECORD resolves before replica storage; the probe must wait across HTTP workers for that storage attempt. +const settledFor = (id) => + new Int32Array(tables.Sharded.auditStore.getUserSharedBuffer(`shard-read-settled:${id}`, new ArrayBuffer(4))); +const storeReplica = tables.Sharded._storeReplica; +tables.Sharded._storeReplica = async function (existingEntry, entry) { + try { + return await storeReplica.call(this, existingEntry, entry); + } finally { + Atomics.add(settledFor(existingEntry.key), 0, 1); + } +}; + +export class SettledReads extends Resource { + get(target) { + return Atomics.load(settledFor(target.id), 0); + } +} + export class LocalRows extends Resource { get(target) { const table = tables[target.get('table')]; - return Array.from(table.primaryStore.getRange({ start: true }), ({ key, value }) => ({ id: key, record: value })); + return Array.from(table.primaryStore.getRange({ start: true }), ({ key }) => { + const entry = table.primaryStore.getEntry(key); + return { + id: key, + record: entry.value, + metadataFlags: entry.metadataFlags, + invalidated: Boolean(entry.metadataFlags & 1), + residencyId: entry.residencyId ?? null, + residency: entry.residencyId ? table.getResidencyRecord(entry.residencyId) : null, + }; + }); } } diff --git a/integrationTests/cluster/staticSharding.test.mjs b/integrationTests/cluster/staticSharding.test.mjs index db422c23a..deddb718d 100644 --- a/integrationTests/cluster/staticSharding.test.mjs +++ b/integrationTests/cluster/staticSharding.test.mjs @@ -281,7 +281,7 @@ suite(`Static sharding (replication pool ${POOL ? 'on' : 'off'})`, { timeout: 60 assert.equal((await get(nodes()[3], `Sharded/${id}`)).body, id); }); - test('limit queries do not fan out to the owning shard, including with pool invalidations (#198)', async () => { + test('limit queries load complete rows from pooled stubs without changing placement (#980)', async () => { const ids = [0, 1, 2, 3, 4].map((i) => `0:limit-${i}`); for (const id of ids) await put(nodes()[0], id, 'limit'); await waitForRows(nodes()[1], ids); @@ -289,9 +289,47 @@ suite(`Static sharding (replication pool ${POOL ? 'on' : 'off'})`, { timeout: 60 const owner = await get(nodes()[0], 'Sharded/?tag=limit&limit(3)'); assert.equal(owner.length, 3); assert.ok(owner.every((row) => ids.includes(row.id))); - await checkNonowners(ids, nodes().slice(2)); - const remote = await get(nodes()[2], 'Sharded/?tag=limit&limit(3)'); - assert.deepEqual(remote, []); + const nonowners = nodes().slice(2); + const query = 'Sharded/?tag=limit&limit(3)'; + const snapshot = () => + Promise.all(nonowners.map(async (node) => (await rawRows(node)).filter((row) => ids.includes(row.id)))); + await checkNonowners(ids, nonowners); + const before = await snapshot(); + const residents = nodes() + .slice(0, 2) + .map((node) => node.hostname) + .sort(); + for (const rows of before) { + assert.deepEqual(rows.map((row) => row.id).sort(), POOL ? [...ids].sort() : []); + if (POOL) { + for (const row of rows) { + assert.equal(row.invalidated, true); + assert.ok(row.residencyId > 0); + assert.deepEqual([...row.residency].sort(), residents); + } + } + } + for (const node of nonowners) assert.deepEqual(await get(node, query, { local: true }), []); + const remote = await get(nonowners[0], query); + if (POOL) { + assert.equal(remote.length, 3); + assert.equal(new Set(remote.map((row) => row.id)).size, 3); + for (const row of remote) { + assert.ok(ids.includes(row.id)); + assert.deepEqual(row, recordFor(row.id, 'limit')); + } + await waitForCondition( + async (signal) => + (await Promise.all(remote.map((row) => get(nonowners[0], `SettledReads/${row.id}`, { signal })))).every( + (count) => count > 0 + ), + { description: 'replica storage attempts for the returned rows to finish' } + ); + } else assert.deepEqual(remote, []); + assert.deepEqual(await snapshot(), before); + await barriers(); + assert.deepEqual(await snapshot(), before); + for (const node of nonowners) assert.deepEqual(await get(node, query, { local: true }), []); assert.equal((await get(nodes()[2], `Sharded/${ids[4]}`)).body, ids[4]); }); diff --git a/replication/DESIGN.md b/replication/DESIGN.md index 31cc2fb9a..43b8a8976 100644 --- a/replication/DESIGN.md +++ b/replication/DESIGN.md @@ -556,7 +556,7 @@ Regressions: `unitTests/replication/replicateFalseSendPaths.test.mjs` (frames), 31. **Point reads need the requested table's decoder, even on a connection without a subscription.** `GET_RECORD` announces the live table's `RecordEncoder` dictionaries before returning its raw value. The encoder owns the storage-engine-specific dictionary lookup; the old primary-store `Symbol.for('structures')` key only worked on LMDB. Announcement state is per requested table id and compares both durable dictionary lengths, table identity, and the table's `schemaVersion`/`schemaDefined` (the receiver takes its schema from this frame, as on the subscription stream; schema reloads edit `attributes` in place, so the array's identity does not change), including empty dictionaries. Unchanged hits check durable counts without rebuilding encoder transitions; repeat misses need no dictionary read. Local encoder arrays alone can lag another worker’s writes. Read an exact-sized binary value (`getBinarySync` on RocksDB, `getBinary` on LMDB): `getBinaryFast` can expose reusable capacity past the record. New requests always include the table name; the responder retains the id/name association for older clients that omit subsequent names, but resolves the live class and `replicate` flag on every request. Blob repair shares a subscription connection, so it uses that peer's announced table id instead of overwriting another table's decoder with a local id. No usable subscription decoder means repair fails before sending. The responder also refuses a mismatched id once that socket has subscribed, protecting the stream from older repair clients that still send their local id. -`integrationTests/cluster/staticSharding.test.mjs` pins two shards with two owners each: local raw keys (including invalidation stubs), remote point reads, same-shard database subscriptions, both owners serving alone, and catch-up after restart. With `replication.threads > 0`, application residency hooks are absent on send workers and nonowners receive indexed stubs (#980); with the pool off, owner-originated static rows leave no nonowner key. Collection queries stay local in both modes, and operations-API writes retain a full row on a nonowner writer with the pool off and on every node with the pool on. These are characterized limitations, not write forwarding or cross-shard query support. Frame-level compatibility and decoding regressions live in `unitTests/replication/replicateFalseSendPaths.test.mjs`. +`integrationTests/cluster/staticSharding.test.mjs` pins two shards with two owners each: local raw keys (including invalidation stubs), remote point reads, same-shard database subscriptions, both owners serving alone, and catch-up after restart. With `replication.threads > 0`, application residency hooks are absent on send workers and nonowners receive indexed stubs (#980); with the pool off, owner-originated static rows leave no nonowner key. Pooled collection queries load complete records from owners for local indexed candidates; local-only queries remain empty and the nonowner stubs retain their flags and residency. Without pooling, the nonowner collection stays empty. Cross-shard limit semantics remain incomplete. Operations-API writes retain a full row on a nonowner writer with the pool off and on every node with the pool on. These are characterized limitations, not write forwarding or cross-shard query support. Frame-level compatibility and decoding regressions live in `unitTests/replication/replicateFalseSendPaths.test.mjs`. ## Tests From ccf0dcc287c4a082f831a645cafcec85a0867d32 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Mon, 26 Oct 2026 07:06:33 -0600 Subject: [PATCH 2/4] test: read shard placement directly from storage Dispatch-Task: harper-pro-main-red-static-sharding-20261022 Co-Authored-By: GPT-5 Codex --- .../cluster/fixture-static-sharding/resources.js | 10 ++++++---- integrationTests/cluster/staticSharding.test.mjs | 8 +++++--- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/integrationTests/cluster/fixture-static-sharding/resources.js b/integrationTests/cluster/fixture-static-sharding/resources.js index 76d16e2d8..491a816db 100644 --- a/integrationTests/cluster/fixture-static-sharding/resources.js +++ b/integrationTests/cluster/fixture-static-sharding/resources.js @@ -3,6 +3,7 @@ for (const table of [tables.Sharded, tables.OtherSharded]) tables.Placed.setResidency((record) => record.homes); // GET_RECORD resolves before replica storage; the probe must wait across HTTP workers for that storage attempt. +// Retrieval storage runs on these fixture-loaded HTTP workers, not the replication pool workers. const settledFor = (id) => new Int32Array(tables.Sharded.auditStore.getUserSharedBuffer(`shard-read-settled:${id}`, new ArrayBuffer(4))); const storeReplica = tables.Sharded._storeReplica; @@ -15,18 +16,19 @@ tables.Sharded._storeReplica = async function (existingEntry, entry) { }; export class SettledReads extends Resource { + static loadAsInstance = false; + get(target) { - return Atomics.load(settledFor(target.id), 0); + return { settled: Atomics.load(settledFor(target.id), 0) }; } } export class LocalRows extends Resource { get(target) { const table = tables[target.get('table')]; - return Array.from(table.primaryStore.getRange({ start: true }), ({ key }) => { - const entry = table.primaryStore.getEntry(key); + return Array.from(table.primaryStore.getRange({ start: true, versions: true }), (entry) => { return { - id: key, + id: entry.key, record: entry.value, metadataFlags: entry.metadataFlags, invalidated: Boolean(entry.metadataFlags & 1), diff --git a/integrationTests/cluster/staticSharding.test.mjs b/integrationTests/cluster/staticSharding.test.mjs index deddb718d..34704f7eb 100644 --- a/integrationTests/cluster/staticSharding.test.mjs +++ b/integrationTests/cluster/staticSharding.test.mjs @@ -281,7 +281,7 @@ suite(`Static sharding (replication pool ${POOL ? 'on' : 'off'})`, { timeout: 60 assert.equal((await get(nodes()[3], `Sharded/${id}`)).body, id); }); - test('limit queries load complete rows from pooled stubs without changing placement (#980)', async () => { + test('limit queries load complete rows from pooled stubs without changing placement (#198, #980)', async () => { const ids = [0, 1, 2, 3, 4].map((i) => `0:limit-${i}`); for (const id of ids) await put(nodes()[0], id, 'limit'); await waitForRows(nodes()[1], ids); @@ -310,18 +310,20 @@ suite(`Static sharding (replication pool ${POOL ? 'on' : 'off'})`, { timeout: 60 } } for (const node of nonowners) assert.deepEqual(await get(node, query, { local: true }), []); + if (POOL) + for (const id of ids) assert.deepStrictEqual(await get(nonowners[0], `SettledReads/${id}`), { settled: 0 }); const remote = await get(nonowners[0], query); if (POOL) { assert.equal(remote.length, 3); assert.equal(new Set(remote.map((row) => row.id)).size, 3); for (const row of remote) { assert.ok(ids.includes(row.id)); - assert.deepEqual(row, recordFor(row.id, 'limit')); + assert.deepStrictEqual(row, recordFor(row.id, 'limit')); } await waitForCondition( async (signal) => (await Promise.all(remote.map((row) => get(nonowners[0], `SettledReads/${row.id}`, { signal })))).every( - (count) => count > 0 + ({ settled }) => settled > 0 ), { description: 'replica storage attempts for the returned rows to finish' } ); From b9ef34c83a5a97b9c01ca9472aa1511112f80ae4 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Mon, 26 Oct 2026 11:32:12 -0600 Subject: [PATCH 3/4] fix: pin core local-only collection completeness repair Keep the strengthened static-sharding assertion and include the coordinated runtime fix from HarperFast/harper#3571. Dispatch-Task: harper-pro-main-red-static-sharding-20261022 Co-Authored-By: GPT-5 Codex --- core | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core b/core index 808d5f543..7cc321be1 160000 --- a/core +++ b/core @@ -1 +1 @@ -Subproject commit 808d5f5438ad3651129bdd61a8945a03cbbafa64 +Subproject commit 7cc321be1934f62ee7d29e74341d8e98d9bd89ca From bd7587f3e9bd2a8d42fb310c26a9a11f063c309c Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Mon, 26 Oct 2026 11:53:20 -0600 Subject: [PATCH 4/4] fix: pin core sorted local-only invalidation repair Include the computed-sort completeness repair in the coordinated core draft HarperFast/harper#3571. Dispatch-Task: harper-pro-main-red-static-sharding-20261022 Co-Authored-By: GPT-5 Codex --- core | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core b/core index 7cc321be1..c1e192c77 160000 --- a/core +++ b/core @@ -1 +1 @@ -Subproject commit 7cc321be1934f62ee7d29e74341d8e98d9bd89ca +Subproject commit c1e192c77522f558de7aa1271835aaa9f143ec8f