Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/workflows/integration-tests.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -476,6 +476,7 @@ jobs:
integrationTests/cluster/recordLockCluster.test.mjs
integrationTests/cluster/cacheReplicationSource.test.mjs
integrationTests/cluster/isolatedApplicationReplication.test.mjs
integrationTests/cluster/staticSharding.test.mjs

- name: Upload Harper server logs
if: failure() || cancelled()
Expand Down
32 changes: 31 additions & 1 deletion integrationTests/cluster/fixture-static-sharding/resources.js
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,39 @@ for (const table of [tables.Sharded, tables.OtherSharded])
table.setResidencyById((id) => Number(id.match(/^(\d+):/)?.[1] ?? 0));
tables.Placed.setResidency((record) => record.homes);

// GET_RECORD resolves before replica storage; the probe must wait across HTTP workers for that storage attempt.
// Retrieval storage runs on these fixture-loaded HTTP workers, not the replication pool workers.
const settledFor = (id) =>
new Int32Array(tables.Sharded.auditStore.getUserSharedBuffer(`shard-read-settled:${id}`, new ArrayBuffer(4)));
Comment thread
kriszyp marked this conversation as resolved.
Resolved
const storeReplica = tables.Sharded._storeReplica;
tables.Sharded._storeReplica = async function (existingEntry, entry) {
try {
return await storeReplica.call(this, existingEntry, entry);
} finally {
Atomics.add(settledFor(existingEntry.key), 0, 1);
}
};
Comment thread
kriszyp marked this conversation as resolved.
Resolved

export class SettledReads extends Resource {
static loadAsInstance = false;

get(target) {
return { settled: Atomics.load(settledFor(target.id), 0) };
}
}

export class LocalRows extends Resource {
get(target) {
const table = tables[target.get('table')];
return Array.from(table.primaryStore.getRange({ start: true }), ({ key, value }) => ({ id: key, record: value }));
return Array.from(table.primaryStore.getRange({ start: true, versions: true }), (entry) => {
return {
id: entry.key,
record: entry.value,
metadataFlags: entry.metadataFlags,
invalidated: Boolean(entry.metadataFlags & 1),
residencyId: entry.residencyId ?? null,
residency: entry.residencyId ? table.getResidencyRecord(entry.residencyId) : null,
};
});
}
}
48 changes: 44 additions & 4 deletions integrationTests/cluster/staticSharding.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -281,17 +281,57 @@ suite(`Static sharding (replication pool ${POOL ? 'on' : 'off'})`, { timeout: 60
assert.equal((await get(nodes()[3], `Sharded/${id}`)).body, id);
});

test('limit queries do not fan out to the owning shard, including with pool invalidations (#198)', async () => {
test('limit queries load complete rows from pooled stubs without changing placement (#198, #980)', async () => {
const ids = [0, 1, 2, 3, 4].map((i) => `0:limit-${i}`);
for (const id of ids) await put(nodes()[0], id, 'limit');
await waitForRows(nodes()[1], ids);
await barriers(nodes().slice(0, 2));
const owner = await get(nodes()[0], 'Sharded/?tag=limit&limit(3)');
assert.equal(owner.length, 3);
assert.ok(owner.every((row) => ids.includes(row.id)));
await checkNonowners(ids, nodes().slice(2));
const remote = await get(nodes()[2], 'Sharded/?tag=limit&limit(3)');
assert.deepEqual(remote, []);
const nonowners = nodes().slice(2);
const query = 'Sharded/?tag=limit&limit(3)';
const snapshot = () =>
Promise.all(nonowners.map(async (node) => (await rawRows(node)).filter((row) => ids.includes(row.id))));
await checkNonowners(ids, nonowners);
const before = await snapshot();
const residents = nodes()
.slice(0, 2)
.map((node) => node.hostname)
.sort();
for (const rows of before) {
assert.deepEqual(rows.map((row) => row.id).sort(), POOL ? [...ids].sort() : []);
if (POOL) {
for (const row of rows) {
assert.equal(row.invalidated, true);
assert.ok(row.residencyId > 0);
assert.deepEqual([...row.residency].sort(), residents);
}
}
}
for (const node of nonowners) assert.deepEqual(await get(node, query, { local: true }), []);
if (POOL)
for (const id of ids) assert.deepStrictEqual(await get(nonowners[0], `SettledReads/${id}`), { settled: 0 });
const remote = await get(nonowners[0], query);
if (POOL) {
assert.equal(remote.length, 3);
assert.equal(new Set(remote.map((row) => row.id)).size, 3);
for (const row of remote) {
assert.ok(ids.includes(row.id));
assert.deepStrictEqual(row, recordFor(row.id, 'limit'));
}
await waitForCondition(
async (signal) =>
(await Promise.all(remote.map((row) => get(nonowners[0], `SettledReads/${row.id}`, { signal })))).every(
({ settled }) => settled > 0
),
{ description: 'replica storage attempts for the returned rows to finish' }
);
} else assert.deepEqual(remote, []);
assert.deepEqual(await snapshot(), before);
await barriers();
assert.deepEqual(await snapshot(), before);
for (const node of nonowners) assert.deepEqual(await get(node, query, { local: true }), []);
assert.equal((await get(nodes()[2], `Sharded/${ids[4]}`)).body, ids[4]);
});

Expand Down
2 changes: 1 addition & 1 deletion replication/DESIGN.md
Original file line number Diff line number Diff line change
Expand Up @@ -568,7 +568,7 @@ Regressions: `unitTests/replication/replicateFalseSendPaths.test.mjs` (frames),

31. **Point reads need the requested table's decoder, even on a connection without a subscription.** `GET_RECORD` announces the live table's `RecordEncoder` dictionaries before returning its raw value. The encoder owns the storage-engine-specific dictionary lookup; the old primary-store `Symbol.for('structures')` key only worked on LMDB. Announcement state is per requested table id and compares both durable dictionary lengths, table identity, and the table's `schemaVersion`/`schemaDefined` (the receiver takes its schema from this frame, as on the subscription stream; schema reloads edit `attributes` in place, so the array's identity does not change), including empty dictionaries. Unchanged hits check durable counts without rebuilding encoder transitions; repeat misses need no dictionary read. Local encoder arrays alone can lag another worker’s writes. Read an exact-sized binary value (`getBinarySync` on RocksDB, `getBinary` on LMDB): `getBinaryFast` can expose reusable capacity past the record. New requests always include the table name; the responder retains the id/name association for older clients that omit subsequent names, but resolves the live class and `replicate` flag on every request. Blob repair shares a subscription connection, so it uses that peer's announced table id instead of overwriting another table's decoder with a local id. No usable subscription decoder means repair fails before sending. The responder also refuses a mismatched id once that socket has subscribed, protecting the stream from older repair clients that still send their local id.

`integrationTests/cluster/staticSharding.test.mjs` pins two shards with two owners each: local raw keys (including invalidation stubs), remote point reads, same-shard database subscriptions, both owners serving alone, and catch-up after restart. With `replication.threads > 0`, application residency hooks are absent on send workers and nonowners receive indexed stubs (#980); with the pool off, owner-originated static rows leave no nonowner key. Collection queries stay local in both modes, and operations-API writes retain a full row on a nonowner writer with the pool off and on every node with the pool on. These are characterized limitations, not write forwarding or cross-shard query support. Frame-level compatibility and decoding regressions live in `unitTests/replication/replicateFalseSendPaths.test.mjs`.
`integrationTests/cluster/staticSharding.test.mjs` pins two shards with two owners each: local raw keys (including invalidation stubs), remote point reads, same-shard database subscriptions, both owners serving alone, and catch-up after restart. With `replication.threads > 0`, application residency hooks are absent on send workers and nonowners receive indexed stubs (#980); with the pool off, owner-originated static rows leave no nonowner key. Pooled collection queries load complete records from owners for local indexed candidates; local-only queries remain empty and the nonowner stubs retain their flags and residency. Without pooling, the nonowner collection stays empty. Cross-shard limit semantics remain incomplete. Operations-API writes retain a full row on a nonowner writer with the pool off and on every node with the pool on. These are characterized limitations, not write forwarding or cross-shard query support. Frame-level compatibility and decoding regressions live in `unitTests/replication/replicateFalseSendPaths.test.mjs`.

## Tests

Expand Down
Loading