From 76327698acf982bc32cd33cc9b64373f8354ffe7 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Wed, 21 Oct 2026 18:25:22 -0600 Subject: [PATCH 01/11] fix: distinguish non-resident peers from missing records Dispatch-Task: hp-residency-not-resident-answer Co-Authored-By: GPT-5 Codex --- DESIGN.md | 2 +- .../resources.js | 6 +- .../cluster/residencyReads.test.mjs | 34 ++++++- replication/DESIGN.md | 21 +++-- replication/protocolCapabilities.ts | 12 ++- replication/replicationConnection.ts | 16 +++- replication/replicator.ts | 27 +++--- .../replication/protocolCapabilities.test.mjs | 25 +++++ unitTests/replication/residencyWire.test.mjs | 92 +++++++++++++++++-- 9 files changed, 200 insertions(+), 35 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index 57c9ac82a..c9768772e 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -1,6 +1,6 @@ # Design notes -- [Send-side residency](replication/DESIGN.md#send-side-residency): every non-resident receives current stubs unless residency is by id; the copy sender and GET_RECORD responder preserve their partial-record status. +- [Send-side residency](replication/DESIGN.md#send-side-residency): every non-resident receives current stubs unless residency is by id; capability-gated copy frames preserve stubs, and GET_RECORD distinguishes not resident from absence. - [A clone copies filesystem-only applications as the leader packaged them](cloneNode/DESIGN.md#a-clone-copies-filesystem-only-applications-as-the-leader-packaged-them): `node_modules` included from a leader that streams it (5.2.3+ over HTTP); an application it cannot copy keeps the clone Unavailable and uncloned. - [Removed-origin history](replication/DESIGN.md#removed-origin-history): membership removal preserves audit history, origin mappings and resume cursors; retention purges eligible segments independently. - [Replication base-copy compatibility](replication/DESIGN.md#base-copy-peer-safety): reported 5.x peers receive historical copies; v4 and unidentified peers retain baseline verification. diff --git a/integrationTests/cluster/fixture-residency-confirmation/resources.js b/integrationTests/cluster/fixture-residency-confirmation/resources.js index a9cf4b2bd..d51f87568 100644 --- a/integrationTests/cluster/fixture-residency-confirmation/resources.js +++ b/integrationTests/cluster/fixture-residency-confirmation/resources.js @@ -16,10 +16,14 @@ export class ResidencyState extends Resource { }; } - async post(target, { removeLocal, prefer, latencySlot, statusSlots }) { + async post(target, { removeLocal, prefer, latencySlot, statusSlots, residencyFor, residents }) { const table = tables.Open; // Deliberately leave peers' residency stubs unchanged, as with a stale residency list. if (removeLocal) await table.primaryStore.remove(removeLocal); + if (residencyFor) { + const { residencyId } = table.primaryStore.getEntry(residencyFor); + await table.dbisDB.put([Symbol.for('residency_by_id'), residencyId], residents); + } if (prefer) { const status = new Float64Array( table.auditStore.getUserSharedBuffer(['replicated', 'data', prefer], new ArrayBuffer(statusSlots * 8)) diff --git a/integrationTests/cluster/residencyReads.test.mjs b/integrationTests/cluster/residencyReads.test.mjs index 3882437f2..d0e137102 100644 --- a/integrationTests/cluster/residencyReads.test.mjs +++ b/integrationTests/cluster/residencyReads.test.mjs @@ -200,7 +200,7 @@ suite('partial replication reads and placement', { timeout: 600_000 }, (ctx) => assert.equal(response.body.body, 'new payload'); }); - test('tries another listed resident after not-held, and returns absence when all answer absent', async () => { + test('tries another listed resident after absence, and returns absence when all answer absent', async () => { for (const id of ['retry', 'absent']) { await put(nodes[0], id, { name: id, body: 'from the remaining resident' }); const states = await placed(id); @@ -221,12 +221,40 @@ suite('partial replication reads and placement', { timeout: 600_000 }, (ctx) => } }); + test('a stale list naming only stub holders fails instead of reporting absence', async () => { + const id = 'stale-list'; + await put(nodes[0], id, { name: id, body: 'still resident elsewhere' }); + const states = await placed(id); + const [reader, otherStub] = states.flatMap((entry, index) => (entry.invalidated ? [index] : [])); + await control(nodes[reader], { residencyFor: id, residents: [names[otherStub]] }); + try { + const response = await waitForCondition( + async (signal) => { + const result = await request(nodes[reader], `Open/${id}`, { signal }); + return (result.status !== 502 || /not resident/i.test(JSON.stringify(result.body))) && result; + }, + { timeoutMs: 30_000, description: 'stale-list holder answers the retrieval request' } + ); + assert.equal(response.status, 502, JSON.stringify(response)); + assert.match(JSON.stringify(response.body), /not resident/i); + const query = await request(nodes[reader], `Open/?name==${id}&select(id,body)`); + assert.ok( + query.status >= 500 || (query.body.length === 1 && /not resident/i.test(query.body[0].message)), + JSON.stringify(query) + ); + assert.ok((await state(nodes[reader], id)).invalidated); + assert.equal((await state(nodes[0], id)).value.body, 'still resident elsewhere'); + } finally { + await control(nodes[reader], { residencyFor: id, residents: states[reader].residency }); + } + }); + test('unreachable residents cause read failures instead of absence or a partial row', async () => { await put(nodes[0], 'unavailable', { name: 'unavailable', body: 'still exists' }); const states = await placed('unavailable'); const residentIndexes = states.flatMap((entry, index) => (entry.invalidated ? [] : [index])); const reader = states.findIndex((entry) => entry.invalidated); - // A successful empty answer is not proof of absence if another listed resident is offline. + // One genuine absence is sufficient once no available holder returned a record. await put( nodes[0], 'partly-unavailable', @@ -238,7 +266,7 @@ suite('partial replication reads and placement', { timeout: 600_000 }, (ctx) => await stopNodeProcess(nodes[residentIndexes[1]]); await disconnected(reader, [residentIndexes[1]]); const partialResponse = await request(nodes[reader], 'Open/partly-unavailable'); - assert.equal(partialResponse.status, 502, JSON.stringify(partialResponse)); + assert.equal(partialResponse.status, 404, JSON.stringify(partialResponse)); await Promise.all(residentIndexes.map((index) => stopNodeProcess(nodes[index]))); await disconnected(reader, residentIndexes); const response = await request(nodes[reader], 'Open/unavailable'); diff --git a/replication/DESIGN.md b/replication/DESIGN.md index 0ef2501bf..7aff29fba 100644 --- a/replication/DESIGN.md +++ b/replication/DESIGN.md @@ -79,6 +79,7 @@ Each socket resolves the peer's bag once per `NODE_NAME` into a frozen object wi | `originFloors` | level | `0` (unsupported) | `min(local, peer)`; advertised as `1` only by a RocksDB build. Gates the floor vector on `SEQUENCE_ID_UPDATE[2]`, its storage as `nodes[].closedFloor`, and a resume request that starts at a stored floor — see Non-obvious behavior 27 | | `tableLifecycle` | level | `0` (unsupported) | `min(local, peer)`; a peer at ≥1 reads lifecycle stamps and the `DB_SCHEMA[4]` drop-marker list, which is sent only to such a peer (harper#1212) | | `residencyStubCopy` | level | `0` (unsupported) | `min(local, peer)`; a peer at ≥1 keeps a full resident record on an equal-version invalidation (harper#3322), so the base copy may send it a stored stub as `invalidate` | +| `residencyNotResident` | level | `0` (unsupported) | `min(local, peer)`; a peer at ≥1 distinguishes a GET_RECORD not-resident answer (`false`) from an empty absence answer | The kinds are not cosmetic. A **level** coerces (`Number(value)`), because the behavior being preserved is `capabilities?.subscriptionSetupAck >= 1`, a coercing comparison that a peer sending `true` or `'1'` passes today. A **parameter** does not coerce, because its predecessor was `Number.isFinite(raw) && raw > 0`, which already rejects `'300'`. A parameter is also never min-clamped: a millisecond budget is not a feature level. @@ -88,7 +89,7 @@ An absent bag resolves to all defaults, which is exactly the pre-registry behavi **Which one do you add?** A capability key for an additive feature, so peers can adopt it independently. `protocolVersion` only for a change to the SHAPE of the protocol. Neither for durable state: `COPY_ORDER_VERSION` stays in the copy cursor because it describes persisted cursor content, not a peer's abilities. -**Sender-side gating discipline.** A sender must not emit a frame type, or a field a receiver must understand, that the peer has not advertised support for. This is the invariant that makes the unknown-frame default case safe to continue past. The gated frames are the record-lock control entries (`lockRelease` / `lockBarrier`, audit action nibbles 12 / 13, `isLockControlType`) and `RECORD_LOCK_HOMES_DIGEST`: `sendAuditRecord` skips a control entry, next to the `LOCAL_ONLY` skip, unless this socket has learned the peer's bag and it carries exactly the local `recordLocks` level (`peerSupportsRecordLocks`), and the skipped entry advances the peer's cursor like any other skipped record — a control entry that never arrives can only cost a requester a 423, never a second holder (see below). The digest announce is gated on the same condition. The base copy sends a stored residency stub as `invalidate` only to a learned peer advertising `residencyStubCopy`; any other peer, including one whose bag is not learned yet, gets the row exactly as before the key existed (a `put` of the stub without the residency-id flag), never a skip: the copy may be that peer's only source for the row. +**Sender-side gating discipline.** A sender must not emit a frame type, or a field a receiver must understand, that the peer has not advertised support for. This is the invariant that makes the unknown-frame default case safe to continue past. The gated frames are the record-lock control entries (`lockRelease` / `lockBarrier`, audit action nibbles 12 / 13, `isLockControlType`) and `RECORD_LOCK_HOMES_DIGEST`: `sendAuditRecord` skips a control entry, next to the `LOCAL_ONLY` skip, unless this socket has learned the peer's bag and it carries exactly the local `recordLocks` level (`peerSupportsRecordLocks`), and the skipped entry advances the peer's cursor like any other skipped record — a control entry that never arrives can only cost a requester a 423, never a second holder (see below). The digest announce is gated on the same condition. GET_RECORD sends `false` for a stored stub only after learning a requester advertising `residencyNotResident`; every other requester receives the identical two-element empty response. The base copy sends a stored residency stub as `invalidate` only to a learned peer advertising `residencyStubCopy`; any other peer, including one whose bag is not learned yet, gets the row exactly as before the key existed (a `put` of the stub without the residency-id flag), never a skip: the copy may be that peer's only source for the row. **Observability.** The frozen result is mirrored on `NodeReplicationConnection.peerCapabilities`, replaced per socket — the slot must stay replaceable, since the object outlives the socket. `undefined` there means NOT YET LEARNED on the current socket, never "the peer does not support it": a peer that advertised nothing is the frozen defaults object, and a consumer gating on a capability must tell the two apart or it will deny a healthy peer mid-reconnect. Nothing is reported before a `NODE_NAME` lands, so a peer that wedges before its handshake is not misreported as pre-#646. The result is carried to the main thread on the existing `connected-to-node` message, deduplicated by field value. `applyConnectionMetadata` fences it with a `(worker thread id, per-worker session ordinal)` pair so a message from a replaced socket cannot overwrite the live entry; a wall clock cannot serve as that fence, since two workers can open a session in the same millisecond and a clock can step backwards. `cluster_status` reports the **effective** values per `database_sockets` entry as `peerCapabilities`, omitted while the shared-memory truth reads the socket down, plus `unknownCommandFrames`, omitted when zero. All of it is observability: the socket-local frozen object is the enforcement authority. @@ -340,10 +341,14 @@ Transport invariant: a stub is never sent, served, or stored as a complete recor copy/retrieval paths, except the base copy to a peer without `residencyStubCopy` (below). `sendAuditRecord` sends every write to every non-resident peer as an invalidation unless residency is by id. The copy walk uses `invalidate` for `INVALIDATED` rows and translates the stored residency flag to the audit flag. GET_RECORD refuses -stubs with an empty response and supplies current decoder structures for full rows. `Replicator.load` -tries the remaining residents after an empty stored-stub response; absence requires every non-self -resident to answer, while an unreachable resident leaves a read error. Id-derived reads retain their -first-answer semantics. Blob repair shares a subscription socket, so its request and response decoder +stubs with `[GET_RECORD_RESPONSE, requestId, false]` for a learned requester advertising +`residencyNotResident`, and supplies current decoder structures for full rows. A missing or deleted +row keeps the two-element empty response. The receiver forwards only strict `false` as not resident, +without decoding or relocation. `Replicator.load` tries remaining available holders after either +negative answer for a stored list; a full record wins, otherwise one genuine empty answer permits +absence even if another holder is unavailable. If every contacted holder is not resident or +unreachable, the read fails with a 502 `ServerError`; the first network error remains its cause. Id-derived reads retry not resident +but retain their first genuine answer semantics. Blob repair shares a subscription socket, so its request and response decoder use the peer's table id; a missing usable decoder fails locally. After a reconnect, an idle table may have no decoder until its next replicated write, so blob repair can fail for that table in the meantime. A subscribed sender refuses mismatched @@ -353,7 +358,11 @@ receiver lacks core's equal-version resident guard and would demote a full recor peers get the pre-gate `put`, which they skip at equal version, and which a receiver without the row stores as complete — today's behavior until it upgrades. The excluded-peer invalidations and the GET_RECORD empty answer stay ungated: older receivers already apply an invalidation to a -non-resident, and read an empty answer as absence without storing anything. +non-resident, and read an empty answer as absence without storing anything. An older responder +still answers a stub as empty, which the requester must count as absence. A lagging holder can also +report absence before receiving the row; this protocol has no version-qualified absence proof. +A genuine empty answer from either can therefore produce 404 when another holder is unavailable. +The per-request wait still depends on the socket watchdog; this change adds no request timer. These transport guards do not repair relocate-on-read (#436/#980): query materialization uses the existing path, whose missing default residency can mark other peers' partial rows complete. The RocksDB relocation audit append can also fail after returning a successful read. diff --git a/replication/protocolCapabilities.ts b/replication/protocolCapabilities.ts index d0ccfe7a5..aab74a316 100644 --- a/replication/protocolCapabilities.ts +++ b/replication/protocolCapabilities.ts @@ -61,6 +61,8 @@ export const TABLE_LIFECYCLE_CAPABILITY = 1; */ export const RESIDENCY_STUB_COPY_CAPABILITY = 1; +export const RESIDENCY_NOT_RESIDENT_CAPABILITY = 1; + /** Effective values for one socket: versions and levels are already `min(local, peer)`. */ export interface ResolvedPeerCapabilities { safeCopyAudit: number; @@ -72,6 +74,7 @@ export interface ResolvedPeerCapabilities { originFloors: number; tableLifecycle: number; residencyStubCopy: number; + residencyNotResident: number; } /** Coerces, because the comparison it replaces did: see the kind table in DESIGN.md. */ @@ -116,6 +119,7 @@ export function resolvePeerCapabilities(bag: any): ResolvedPeerCapabilities { originFloors: resolveLevel(bag?.originFloors, ORIGIN_FLOORS_CAPABILITY, 0), tableLifecycle: resolveLevel(bag?.tableLifecycle, TABLE_LIFECYCLE_CAPABILITY, 0), residencyStubCopy: resolveLevel(bag?.residencyStubCopy, RESIDENCY_STUB_COPY_CAPABILITY, 0), + residencyNotResident: resolveLevel(bag?.residencyNotResident, RESIDENCY_NOT_RESIDENT_CAPABILITY, 0), }); } @@ -127,6 +131,10 @@ export function peerSupportsResidencyStubCopy(resolved: ResolvedPeerCapabilities return resolved.residencyStubCopy >= RESIDENCY_STUB_COPY_CAPABILITY; } +export function peerSupportsResidencyNotResident(resolved: ResolvedPeerCapabilities): boolean { + return resolved.residencyNotResident >= RESIDENCY_NOT_RESIDENT_CAPABILITY; +} + /** The only reader of the `recordLocks` level: the send gate and the participant set both go through here. */ export function peerSupportsRecordLocks(resolved: ResolvedPeerCapabilities): boolean { return resolved.recordLocks === RECORD_LOCKS_CAPABILITY; @@ -177,6 +185,7 @@ export function buildLocalCapabilities( originFloors: perOriginLogs ? ORIGIN_FLOORS_CAPABILITY : 0, tableLifecycle: TABLE_LIFECYCLE_CAPABILITY, residencyStubCopy: RESIDENCY_STUB_COPY_CAPABILITY, + residencyNotResident: RESIDENCY_NOT_RESIDENT_CAPABILITY, }); } @@ -192,7 +201,8 @@ export function samePeerCapabilities(a: ResolvedPeerCapabilities | undefined, b: a.originCursors === b.originCursors && a.originFloors === b.originFloors && a.tableLifecycle === b.tableLifecycle && - a.residencyStubCopy === b.residencyStubCopy + a.residencyStubCopy === b.residencyStubCopy && + a.residencyNotResident === b.residencyNotResident ); } diff --git a/replication/replicationConnection.ts b/replication/replicationConnection.ts index fe3c40a8b..553c824df 100644 --- a/replication/replicationConnection.ts +++ b/replication/replicationConnection.ts @@ -102,6 +102,7 @@ import { peerSupportsOriginCursors, peerSupportsOriginFloors, peerSupportsRecordLocks, + peerSupportsResidencyNotResident, peerSupportsResidencyStubCopy, peerSupportsTableLifecycle, resolvePeerCapabilities, @@ -5980,7 +5981,13 @@ export function replicateOverWS(ws: ReplicationWebSocket, options: any, authoriz let valueBuffer = table.primaryStore.decoder.decode(binaryEntry, { valueAsBuffer: true }); const entry: any = lastMetadata || {}; if (entry[METADATA] & INVALIDATED) { - ws.send(encode([GET_RECORD_RESPONSE, requestId])); + ws.send( + encode( + peerCapabilitiesLearned && peerSupportsResidencyNotResident(peerCapabilities) + ? [GET_RECORD_RESPONSE, requestId, false] + : [GET_RECORD_RESPONSE, requestId] + ) + ); break; } // RocksDB carries the version inline; unwrap its metadata's value bytes. @@ -5991,6 +5998,11 @@ export function replicateOverWS(ws: ReplicationWebSocket, options: any, authoriz } else { entry.version = recordVersion; } + // A stored deletion is msgpack null after its metadata prefix. + if (valueBuffer.length === 1 && valueBuffer[0] === 0xc0) { + ws.send(encode([GET_RECORD_RESPONSE, requestId])); + break; + } if (entry[METADATA] & HAS_BLOBS) { // if there are blobs, we need to find them and send their contents // but first, the decoding process can destroy our buffer above, so we need to copy it @@ -6066,7 +6078,7 @@ export function replicateOverWS(ws: ReplicationWebSocket, options: any, authoriz return localBlob; } ); - } else resolve(); + } else resolve(entry === false ? false : undefined); awaitingResponse.delete(message[1]); break; } diff --git a/replication/replicator.ts b/replication/replicator.ts index e98a66c01..dc18fa11e 100644 --- a/replication/replicator.ts +++ b/replication/replicator.ts @@ -485,6 +485,8 @@ export function setReplicator(dbName: string, table: any, options: any) { entry.residency || (await table.dbisDB.get([Symbol.for('residency_by_id'), residencyId])); if (residency) { let firstError: Error; + let reportedAbsence = false; + let reportedNotResident = false; const attemptedNodes = new Set(); do { // This loop is for trying multiple nodes if the first one fails. With each iteration, we add the node to the attemptedConnections, @@ -508,18 +510,16 @@ export function setReplicator(dbName: string, table: any, options: any) { } } } - // if there are no connections left, throw an error if (!bestConnection) { - if ( - !entry.residency && - !firstError && - attemptedNodes.size > 0 && - residency.every((name) => name === server.hostname || attemptedNodes.has(name)) - ) - return; - throw ( - firstError || new ServerError(`No connection to any other nodes are available: ${residency}`, 502) + if (reportedAbsence) return; + const error = new ServerError( + reportedNotResident + ? `Listed nodes are not resident or unavailable: ${residency}` + : `No connection to any other nodes are available: ${residency}`, + 502 ); + if (firstError) error.cause = firstError; + throw error; } const request = { requestId: nextId++, @@ -530,7 +530,12 @@ export function setReplicator(dbName: string, table: any, options: any) { attemptedNodes.add(bestNode); try { const found = await bestConnection.getRecord(request); - if (found || entry.residency) return found; + if (found) return found; + if (found === false) reportedNotResident = true; + else { + if (entry.residency) return; + reportedAbsence = true; + } } catch (error) { // if we are still connected, must be a non-network error if (bestConnection.isConnected) throw error; diff --git a/unitTests/replication/protocolCapabilities.test.mjs b/unitTests/replication/protocolCapabilities.test.mjs index d5b24fd16..9b1557622 100644 --- a/unitTests/replication/protocolCapabilities.test.mjs +++ b/unitTests/replication/protocolCapabilities.test.mjs @@ -40,6 +40,7 @@ describe('resolvePeerCapabilities — absent and legacy shapes', () => { originFloors: 0, tableLifecycle: 0, residencyStubCopy: 0, + residencyNotResident: 0, } ); }); @@ -63,6 +64,7 @@ describe('resolvePeerCapabilities — absent and legacy shapes', () => { 'originFloors', 'protocolVersion', 'recordLocks', + 'residencyNotResident', 'residencyStubCopy', 'safeCopyAudit', 'subscriptionSetupAck', @@ -242,6 +244,7 @@ describe('buildLocalCapabilities / the advertised NODE_NAME frame', () => { originFloors: ORIGIN_FLOORS_CAPABILITY, tableLifecycle: TABLE_LIFECYCLE_CAPABILITY, residencyStubCopy: RESIDENCY_STUB_COPY_CAPABILITY, + residencyNotResident: 1, } ); assert.strictEqual(Object.isFrozen(local), true); @@ -284,6 +287,7 @@ describe('buildLocalCapabilities / the advertised NODE_NAME frame', () => { originFloors: ORIGIN_FLOORS_CAPABILITY, tableLifecycle: TABLE_LIFECYCLE_CAPABILITY, residencyStubCopy: RESIDENCY_STUB_COPY_CAPABILITY, + residencyNotResident: 1, } ); }); @@ -478,3 +482,24 @@ describe('residencyStubCopy', () => { assert.strictEqual(samePeerCapabilities(full, without), false); }); }); + +describe('residencyNotResident', () => { + it('defaults absent and invalid levels to unsupported', () => { + for (const value of [undefined, null, 0, -1, 'no', {}, { toString: 0 }]) + assert.equal(resolvePeerCapabilities({ residencyNotResident: value }).residencyNotResident, 0); + }); + + it('advertises and clamps the additive level without changing the protocol version', () => { + const local = buildLocalCapabilities(1, false, false); + assert.equal(local.residencyNotResident, 1); + assert.equal(local.protocolVersion, 2); + for (const value of [1, 5, '1']) + assert.equal(resolvePeerCapabilities({ residencyNotResident: value }).residencyNotResident, 1); + }); + + it('reports a change in residencyNotResident alone', () => { + const full = resolvePeerCapabilities(buildLocalCapabilities(90_000, true, true)); + const without = resolvePeerCapabilities({ ...buildLocalCapabilities(90_000, true, true), residencyNotResident: 0 }); + assert.equal(samePeerCapabilities(full, without), false); + }); +}); diff --git a/unitTests/replication/residencyWire.test.mjs b/unitTests/replication/residencyWire.test.mjs index d87f70e9b..a2238903a 100644 --- a/unitTests/replication/residencyWire.test.mjs +++ b/unitTests/replication/residencyWire.test.mjs @@ -34,8 +34,10 @@ const PEER = 'residency-wire-peer'; class Socket extends EventEmitter { readyState = 1; sent = []; + rawSent = []; _socket = { bytesRead: 0, bytesWritten: 0, setMaxListeners() {} }; send(data) { + this.rawSent.push(Buffer.from(data)); this.sent.push(data[0] > 127 ? decode(Buffer.from(data)) : Buffer.from(data)); } close(code = 1000, reason = '') { @@ -74,6 +76,8 @@ describe('residency stubs on the replication wire', function () { Records.setResidency(() => [PEER]); await Records.put('stub', { id: 'stub', name: 'indexed', body: 'complete' }); await Records.put('full', { id: 'full', name: 'indexed', body: 'complete' }); + await Records.put('deleted', { id: 'deleted', name: 'indexed', body: 'complete' }); + await Records.delete('deleted'); residencyId = Records.primaryStore.getEntry('stub').residencyId; const context = {}; await transaction(context, async () => { @@ -101,8 +105,22 @@ describe('residency stubs on the replication wire', function () { afterEach(() => socket?.close()); - for (const mode of ['retry', 'absent', 'unreachable', 'id-derived']) { - it(`handles ${mode} after the first resident answers not held`, async () => { + for (const [mode, answers, expected] of [ + ['retry', [false, 'full'], 'full'], + ['absent', [false, undefined], undefined], + ['absent-first', [undefined, false], undefined], + ['absent-then-full', [undefined, 'full'], 'full'], + ['not-resident', [false, false], 'error'], + ['unreachable', [false, 'offline'], 'error'], + ['absent-unreachable', [undefined, 'offline'], undefined], + ['not-resident-disconnect', [false, 'disconnect'], 'error'], + ['disconnect-not-resident', ['disconnect', false], 'error'], + ['absent-disconnect', [undefined, 'disconnect'], undefined], + ['id-derived', [false, 'full'], 'full'], + ['id-derived-absent-first', [undefined, 'full'], undefined], + ['id-derived-not-resident', [false, false], 'error'], + ]) { + it(`handles ${mode} across listed residents`, async () => { const peers = [`residency-${mode}-first`, `residency-${mode}-second`]; const contacted = []; const connections = []; @@ -115,11 +133,16 @@ describe('residency stubs on the replication wire', function () { })); sandbox.stub(NodeReplicationConnection.prototype, 'connect').callsFake(function () { connections.push(this); - this.isConnected = !(mode === 'unreachable' && this.nodeName === peers[1]); + this.isConnected = answers[peers.indexOf(this.nodeName)] !== 'offline'; }); sandbox.stub(NodeReplicationConnection.prototype, 'getRecord').callsFake(async function () { contacted.push(this.nodeName); - return this.nodeName === peers[1] && mode === 'retry' ? complete : undefined; + const answer = answers[peers.indexOf(this.nodeName)]; + if (answer === 'disconnect') { + this.isConnected = false; + throw new Error('test peer disconnected'); + } + return answer === 'full' ? complete : answer; }); for (const [index, name] of peers.entries()) getReplicationSharedStatus(Records.auditStore, DATABASE, name)[LATENCY_POSITION] = index; @@ -127,11 +150,15 @@ describe('residency stubs on the replication wire', function () { const pending = ReplicationSource.load({ key: mode, residencyId: 789, - ...(mode === 'id-derived' ? { residency: peers } : {}), + ...(mode.startsWith('id-derived') ? { residency: peers } : {}), }); - if (mode === 'unreachable') await assert.rejects(pending, (error) => error.statusCode === 502); - else assert.equal(await pending, mode === 'retry' ? complete : undefined); - assert.deepEqual(contacted, mode === 'unreachable' || mode === 'id-derived' ? peers.slice(0, 1) : peers); + if (expected === 'error') + await assert.rejects(pending, (error) => error.statusCode === 502 && /not resident/i.test(error.message)); + else assert.equal(await pending, expected === 'full' ? complete : undefined); + assert.deepEqual( + contacted, + answers[1] === 'offline' || mode === 'id-derived-absent-first' ? peers.slice(0, 1) : peers + ); } finally { for (const connection of connections) { connection.isFinished = true; @@ -142,6 +169,51 @@ describe('residency stubs on the replication wire', function () { }); } + it('receives not-resident without a decoder or relocation and keeps absence distinct', async () => { + const relocate = sinon.spy(Records, '_recordRelocate'); + try { + for (const answer of [false, undefined]) { + const pending = session.getRecord({ table: Records, id: 'stub' }); + const request = socket.sent.filter((frame) => frame[0] === GET_RECORD).at(-1); + socket.emit( + 'message', + encode(answer === false ? [GET_RECORD_RESPONSE, request[1], false] : [GET_RECORD_RESPONSE, request[1]]) + ); + assert.equal(await pending, answer); + } + assert.equal(relocate.callCount, 0); + } finally { + relocate.restore(); + } + }); + + for (const peer of ['pre-key', 'unlearned']) { + it(`sends byte-identical legacy empty responses to a ${peer} requester`, async () => { + socket.close(); + socket = new Socket(); + session = replicateOverWS(socket, { database: DATABASE }, { name: PEER, replicates: true }); + if (peer === 'pre-key') { + const bag = { ...buildLocalCapabilities(90_000, false, true) }; + delete bag.residencyNotResident; + socket.emit('message', encode([NODE_NAME, PEER, DATABASE, [], bag])); + await until(() => socket.sent.some((frame) => frame[0] === NODE_NAME)); + } + for (const [index, id] of ['stub', 'missing', 'deleted'].entries()) { + socket.emit('message', encode([GET_RECORD, index, Records.tableId, id, Records.tableName])); + await until(() => socket.sent.some((frame) => frame[0] === GET_RECORD_RESPONSE && frame[1] === index)); + const bytes = socket.rawSent.find((frame) => { + const decoded = decode(frame); + return decoded[0] === GET_RECORD_RESPONSE && decoded[1] === index; + }); + assert.ok(bytes.equals(encode([GET_RECORD_RESPONSE, index]))); + } + const pending = session.getRecord({ table: Records, id: 'missing' }); + const request = socket.sent.filter((frame) => frame[0] === GET_RECORD).at(-1); + socket.emit('message', encode([GET_RECORD_RESPONSE, request[1]])); + assert.equal(await pending, undefined, 'an older responder still reports absence'); + }); + } + it('names the table on repeated requests so each can resolve the live table', async () => { for (let index = 0; index < 2; index++) { const pending = session.getRecord({ table: Records, id: 'missing' }); @@ -194,7 +266,7 @@ describe('residency stubs on the replication wire', function () { assert.equal(socket.sent.find((frame) => frame[0] === GET_RECORD_RESPONSE && frame[1] === 92)[2], undefined); }); - for (const id of ['stub', 'missing', 'full']) { + for (const id of ['stub', 'missing', 'deleted', 'full']) { it(`answers GET_RECORD for ${id} without promoting partial data`, async () => { socket.emit('message', encode([GET_RECORD, 7, Records.tableId, id, Records.tableName])); await until(() => socket.sent.some((frame) => frame[0] === GET_RECORD_RESPONSE)); @@ -205,7 +277,7 @@ describe('residency stubs on the replication wire', function () { const StructonPackr = createStructon(Packr); const decoder = new StructonPackr({ typedStructs: structure.typedStructs, structures: structure.structures }); assert.equal(decoder.decode(response[2].value).body, 'complete'); - } else assert.equal(response[2], undefined, 'a stub or missing row must answer not held'); + } else assert.equal(response[2], id === 'stub' ? false : undefined); }); } From f8f8996e4001ca5609d011828c84ee6a81d2c675 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Wed, 21 Oct 2026 18:25:22 -0600 Subject: [PATCH 02/11] chore: sync core residency transport documentation Dispatch-Task: hp-residency-not-resident-answer Co-Authored-By: GPT-5 Codex --- core | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core b/core index 489eebcf0..929682f01 160000 --- a/core +++ b/core @@ -1 +1 @@ -Subproject commit 489eebcf02c35d9b66fa75e8ec7b226aa92348ea +Subproject commit 929682f017b6c4eaa297a992fdef1c7186095f9d From 7ac013611ecb15869c47b74ef80214e840e8527a Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Wed, 21 Oct 2026 18:37:21 -0600 Subject: [PATCH 03/11] style: format residency capability documentation Dispatch-Task: hp-residency-not-resident-answer Co-Authored-By: GPT-5 Codex --- replication/DESIGN.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/replication/DESIGN.md b/replication/DESIGN.md index 7aff29fba..adb375c99 100644 --- a/replication/DESIGN.md +++ b/replication/DESIGN.md @@ -79,7 +79,7 @@ Each socket resolves the peer's bag once per `NODE_NAME` into a frozen object wi | `originFloors` | level | `0` (unsupported) | `min(local, peer)`; advertised as `1` only by a RocksDB build. Gates the floor vector on `SEQUENCE_ID_UPDATE[2]`, its storage as `nodes[].closedFloor`, and a resume request that starts at a stored floor — see Non-obvious behavior 27 | | `tableLifecycle` | level | `0` (unsupported) | `min(local, peer)`; a peer at ≥1 reads lifecycle stamps and the `DB_SCHEMA[4]` drop-marker list, which is sent only to such a peer (harper#1212) | | `residencyStubCopy` | level | `0` (unsupported) | `min(local, peer)`; a peer at ≥1 keeps a full resident record on an equal-version invalidation (harper#3322), so the base copy may send it a stored stub as `invalidate` | -| `residencyNotResident` | level | `0` (unsupported) | `min(local, peer)`; a peer at ≥1 distinguishes a GET_RECORD not-resident answer (`false`) from an empty absence answer | +| `residencyNotResident` | level | `0` (unsupported) | `min(local, peer)`; a peer at ≥1 distinguishes a GET_RECORD not-resident answer (`false`) from an empty absence answer | The kinds are not cosmetic. A **level** coerces (`Number(value)`), because the behavior being preserved is `capabilities?.subscriptionSetupAck >= 1`, a coercing comparison that a peer sending `true` or `'1'` passes today. A **parameter** does not coerce, because its predecessor was `Number.isFinite(raw) && raw > 0`, which already rejects `'300'`. A parameter is also never min-clamped: a millisecond budget is not a feature level. From 00189c37723bb5f4ec999cb97e5013e952b10ada Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Fri, 23 Oct 2026 06:00:10 -0600 Subject: [PATCH 04/11] fix: require every listed holder to answer before reporting absence Dispatch-Task: hp-residency-not-resident-answer Co-Authored-By: GPT-5 Codex --- .../cluster/residencyReads.test.mjs | 23 +++++++++++++------ replication/DESIGN.md | 11 +++++---- replication/replicator.ts | 14 +++++------ unitTests/replication/residencyWire.test.mjs | 19 +++++++++++---- 4 files changed, 43 insertions(+), 24 deletions(-) diff --git a/integrationTests/cluster/residencyReads.test.mjs b/integrationTests/cluster/residencyReads.test.mjs index d0e137102..cfa63d87d 100644 --- a/integrationTests/cluster/residencyReads.test.mjs +++ b/integrationTests/cluster/residencyReads.test.mjs @@ -249,12 +249,11 @@ suite('partial replication reads and placement', { timeout: 600_000 }, (ctx) => } }); - test('unreachable residents cause read failures instead of absence or a partial row', async () => { + test('unreachable residents cause read failures instead of absence or a partial row', async (t) => { await put(nodes[0], 'unavailable', { name: 'unavailable', body: 'still exists' }); const states = await placed('unavailable'); const residentIndexes = states.flatMap((entry, index) => (entry.invalidated ? [] : [index])); const reader = states.findIndex((entry) => entry.invalidated); - // One genuine absence is sufficient once no available holder returned a record. await put( nodes[0], 'partly-unavailable', @@ -262,11 +261,21 @@ suite('partial replication reads and placement', { timeout: 600_000 }, (ctx) => residentIndexes.map((i) => names[i]).join(',') ); await placed('partly-unavailable'); - await control(nodes[residentIndexes[0]], { removeLocal: 'partly-unavailable' }); - await stopNodeProcess(nodes[residentIndexes[1]]); - await disconnected(reader, [residentIndexes[1]]); - const partialResponse = await request(nodes[reader], 'Open/partly-unavailable'); - assert.equal(partialResponse.status, 404, JSON.stringify(partialResponse)); + await t.test('an empty holder cannot hide the record on a stopped holder', async () => { + const [emptyHolder, stoppedHolder] = residentIndexes; + await control(nodes[emptyHolder], { removeLocal: 'partly-unavailable' }); + assert.equal((await state(nodes[emptyHolder], 'partly-unavailable')).present, false); + const stored = await state(nodes[stoppedHolder], 'partly-unavailable'); + assert.equal(stored.invalidated, false); + assert.equal(stored.value.body, 'still exists'); + assert.ok((await state(nodes[reader], 'partly-unavailable')).invalidated); + await control(nodes[reader], { prefer: names[emptyHolder] }); + await stopNodeProcess(nodes[stoppedHolder]); + await disconnected(reader, [stoppedHolder]); + const response = await request(nodes[reader], 'Open/partly-unavailable'); + assert.equal(response.status, 502, JSON.stringify(response)); + assert.ok((await state(nodes[reader], 'partly-unavailable')).invalidated); + }); await Promise.all(residentIndexes.map((index) => stopNodeProcess(nodes[index]))); await disconnected(reader, residentIndexes); const response = await request(nodes[reader], 'Open/unavailable'); diff --git a/replication/DESIGN.md b/replication/DESIGN.md index adb375c99..b9d033a38 100644 --- a/replication/DESIGN.md +++ b/replication/DESIGN.md @@ -345,10 +345,10 @@ stubs with `[GET_RECORD_RESPONSE, requestId, false]` for a learned requester adv `residencyNotResident`, and supplies current decoder structures for full rows. A missing or deleted row keeps the two-element empty response. The receiver forwards only strict `false` as not resident, without decoding or relocation. `Replicator.load` tries remaining available holders after either -negative answer for a stored list; a full record wins, otherwise one genuine empty answer permits -absence even if another holder is unavailable. If every contacted holder is not resident or -unreachable, the read fails with a 502 `ServerError`; the first network error remains its cause. Id-derived reads retry not resident -but retain their first genuine answer semantics. Blob repair shares a subscription socket, so its request and response decoder +negative answer for stored and id-derived lists. A full record wins. Otherwise absence requires +every listed non-self holder to answer and at least one genuine empty answer. Any unreachable +holder keeps the result at 502; all not-resident answers produce a 502 naming that condition. +The first network error remains the `ServerError` cause. Blob repair shares a subscription socket, so its request and response decoder use the peer's table id; a missing usable decoder fails locally. After a reconnect, an idle table may have no decoder until its next replicated write, so blob repair can fail for that table in the meantime. A subscribed sender refuses mismatched @@ -361,7 +361,8 @@ GET_RECORD empty answer stay ungated: older receivers already apply an invalidat non-resident, and read an empty answer as absence without storing anything. An older responder still answers a stub as empty, which the requester must count as absence. A lagging holder can also report absence before receiving the row; this protocol has no version-qualified absence proof. -A genuine empty answer from either can therefore produce 404 when another holder is unavailable. +Requiring every listed holder to answer prevents an empty reply from hiding a record on an +unreachable holder. It cannot prove absence outside the listed holders, or distinguish legacy stubs. The per-request wait still depends on the socket watchdog; this change adds no request timer. These transport guards do not repair relocate-on-read (#436/#980): query materialization uses the existing path, whose missing default residency can mark other peers' partial rows complete. The diff --git a/replication/replicator.ts b/replication/replicator.ts index dc18fa11e..236fcf676 100644 --- a/replication/replicator.ts +++ b/replication/replicator.ts @@ -511,10 +511,13 @@ export function setReplicator(dbName: string, table: any, options: any) { } } if (!bestConnection) { - if (reportedAbsence) return; + const allAnswered = + !firstError && + residency.every((nodeName) => nodeName === server.hostname || attemptedNodes.has(nodeName)); + if (allAnswered && reportedAbsence) return; const error = new ServerError( - reportedNotResident - ? `Listed nodes are not resident or unavailable: ${residency}` + allAnswered && reportedNotResident + ? `Listed nodes are not resident: ${residency}` : `No connection to any other nodes are available: ${residency}`, 502 ); @@ -532,10 +535,7 @@ export function setReplicator(dbName: string, table: any, options: any) { const found = await bestConnection.getRecord(request); if (found) return found; if (found === false) reportedNotResident = true; - else { - if (entry.residency) return; - reportedAbsence = true; - } + else reportedAbsence = true; } catch (error) { // if we are still connected, must be a non-network error if (bestConnection.isConnected) throw error; diff --git a/unitTests/replication/residencyWire.test.mjs b/unitTests/replication/residencyWire.test.mjs index a2238903a..a8f5e5787 100644 --- a/unitTests/replication/residencyWire.test.mjs +++ b/unitTests/replication/residencyWire.test.mjs @@ -112,12 +112,14 @@ describe('residency stubs on the replication wire', function () { ['absent-then-full', [undefined, 'full'], 'full'], ['not-resident', [false, false], 'error'], ['unreachable', [false, 'offline'], 'error'], - ['absent-unreachable', [undefined, 'offline'], undefined], + ['absent-unreachable', [undefined, 'offline'], 'error'], ['not-resident-disconnect', [false, 'disconnect'], 'error'], ['disconnect-not-resident', ['disconnect', false], 'error'], - ['absent-disconnect', [undefined, 'disconnect'], undefined], + ['absent-disconnect', [undefined, 'disconnect'], 'error'], ['id-derived', [false, 'full'], 'full'], - ['id-derived-absent-first', [undefined, 'full'], undefined], + ['id-derived-absent-first', [undefined, 'full'], 'full'], + ['id-derived-absent-unreachable', [undefined, 'offline'], 'error'], + ['id-derived-absent', [undefined, false], undefined], ['id-derived-not-resident', [false, false], 'error'], ]) { it(`handles ${mode} across listed residents`, async () => { @@ -153,11 +155,18 @@ describe('residency stubs on the replication wire', function () { ...(mode.startsWith('id-derived') ? { residency: peers } : {}), }); if (expected === 'error') - await assert.rejects(pending, (error) => error.statusCode === 502 && /not resident/i.test(error.message)); + await assert.rejects(pending, (error) => { + assert.equal(error.statusCode, 502); + assert.match( + error.message, + answers.includes('offline') || answers.includes('disconnect') ? /No connection/ : /not resident/i + ); + return true; + }); else assert.equal(await pending, expected === 'full' ? complete : undefined); assert.deepEqual( contacted, - answers[1] === 'offline' || mode === 'id-derived-absent-first' ? peers.slice(0, 1) : peers + peers.filter((_, index) => answers[index] !== 'offline') ); } finally { for (const connection of connections) { From 6c942380b7df51d4de300ff818e33108ab0b4694 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Fri, 23 Oct 2026 06:39:56 -0600 Subject: [PATCH 05/11] chore: sync clarified core residency documentation Dispatch-Task: hp-residency-not-resident-answer Co-Authored-By: GPT-5 Codex --- core | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core b/core index 929682f01..7fbacfa47 160000 --- a/core +++ b/core @@ -1 +1 @@ -Subproject commit 929682f017b6c4eaa297a992fdef1c7186095f9d +Subproject commit 7fbacfa472dc87d0659dbbfbb14be207754777ac From a7517a48d81290c6e5fdbe96b6aaea3e46b9346e Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Fri, 23 Oct 2026 06:39:56 -0600 Subject: [PATCH 06/11] docs: clarify legacy stub copies still follow residency routing Dispatch-Task: hp-residency-not-resident-answer Co-Authored-By: GPT-5 Codex --- replication/DESIGN.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/replication/DESIGN.md b/replication/DESIGN.md index b9d033a38..c2a3df359 100644 --- a/replication/DESIGN.md +++ b/replication/DESIGN.md @@ -353,10 +353,11 @@ use the peer's table id; a missing usable decoder fails locally. After a reconne have no decoder until its next replicated write, so blob repair can fail for that table in the meantime. A subscribed sender refuses mismatched table ids before announcing structures; the read is synchronous, so nothing changes between that check and the send. Tests: `residencyWire.test.mjs` and the four-node `residencyReads.test.mjs`. -The copy sends a stub as `invalidate` only to a peer advertising `residencyStubCopy`: an older +The copy walk encodes a stub as `invalidate` only for a peer advertising `residencyStubCopy`: an older receiver lacks core's equal-version resident guard and would demote a full record it holds. Other -peers get the pre-gate `put`, which they skip at equal version, and which a receiver without the row -stores as complete — today's behavior until it upgrades. The excluded-peer invalidations and the +peers retain the pre-gate `put` encoding. Normal residency routing still invalidates or skips excluded +peers. For a listed peer (or when no residency list is available), the legacy `put` is skipped at +equal version; a receiver without the row stores the projection as complete — today's behavior until it upgrades. The excluded-peer invalidations and the GET_RECORD empty answer stay ungated: older receivers already apply an invalidation to a non-resident, and read an empty answer as absence without storing anything. An older responder still answers a stub as empty, which the requester must count as absence. A lagging holder can also From 90be42028d0e40848e01c8c5c247f1483c5fdee2 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Fri, 23 Oct 2026 07:13:54 -0600 Subject: [PATCH 07/11] test: prove the surviving residency holder answers before outage assertions Use a separate single-holder missing key to establish a real empty reply, then repeat the stopped-holder read. Tighten strict-false receiver coverage and capability expectations, and describe the enduring request-timer limitation. Dispatch-Task: hp-residency-not-resident-answer Co-Authored-By: GPT-5 Codex --- integrationTests/cluster/residencyReads.test.mjs | 13 +++++++++++-- replication/DESIGN.md | 2 +- unitTests/replication/protocolCapabilities.test.mjs | 5 +++-- unitTests/replication/residencyWire.test.mjs | 8 ++++---- 4 files changed, 19 insertions(+), 9 deletions(-) diff --git a/integrationTests/cluster/residencyReads.test.mjs b/integrationTests/cluster/residencyReads.test.mjs index cfa63d87d..2fe16fe31 100644 --- a/integrationTests/cluster/residencyReads.test.mjs +++ b/integrationTests/cluster/residencyReads.test.mjs @@ -270,10 +270,19 @@ suite('partial replication reads and placement', { timeout: 600_000 }, (ctx) => assert.equal(stored.value.body, 'still exists'); assert.ok((await state(nodes[reader], 'partly-unavailable')).invalidated); await control(nodes[reader], { prefer: names[emptyHolder] }); + await put(nodes[emptyHolder], 'partly-empty-probe', { body: 'probe' }, names[emptyHolder]); + await placed('partly-empty-probe'); + await control(nodes[emptyHolder], { removeLocal: 'partly-empty-probe' }); await stopNodeProcess(nodes[stoppedHolder]); await disconnected(reader, [stoppedHolder]); - const response = await request(nodes[reader], 'Open/partly-unavailable'); - assert.equal(response.status, 502, JSON.stringify(response)); + for (let attempt = 0; attempt < 8; attempt++) { + await waitForCondition( + async (signal) => (await request(nodes[reader], 'Open/partly-empty-probe', { signal })).status === 404, + { timeoutMs: 30_000, description: 'reader receives an empty answer from the surviving holder' } + ); + const response = await request(nodes[reader], 'Open/partly-unavailable'); + assert.equal(response.status, 502, JSON.stringify(response)); + } assert.ok((await state(nodes[reader], 'partly-unavailable')).invalidated); }); await Promise.all(residentIndexes.map((index) => stopNodeProcess(nodes[index]))); diff --git a/replication/DESIGN.md b/replication/DESIGN.md index c2a3df359..e3779faeb 100644 --- a/replication/DESIGN.md +++ b/replication/DESIGN.md @@ -364,7 +364,7 @@ still answers a stub as empty, which the requester must count as absence. A lagg report absence before receiving the row; this protocol has no version-qualified absence proof. Requiring every listed holder to answer prevents an empty reply from hiding a record on an unreachable holder. It cannot prove absence outside the listed holders, or distinguish legacy stubs. -The per-request wait still depends on the socket watchdog; this change adds no request timer. +GET_RECORD has no per-request timer; its wait depends on the socket watchdog. These transport guards do not repair relocate-on-read (#436/#980): query materialization uses the existing path, whose missing default residency can mark other peers' partial rows complete. The RocksDB relocation audit append can also fail after returning a successful read. diff --git a/unitTests/replication/protocolCapabilities.test.mjs b/unitTests/replication/protocolCapabilities.test.mjs index 9b1557622..73146d1a7 100644 --- a/unitTests/replication/protocolCapabilities.test.mjs +++ b/unitTests/replication/protocolCapabilities.test.mjs @@ -9,6 +9,7 @@ import { ORIGIN_FLOORS_CAPABILITY, RECORD_LOCKS_CAPABILITY, RESIDENCY_STUB_COPY_CAPABILITY, + RESIDENCY_NOT_RESIDENT_CAPABILITY, SUBSCRIPTION_SETUP_ACK_CAPABILITY, TABLE_LIFECYCLE_CAPABILITY, buildLocalCapabilities, @@ -244,7 +245,7 @@ describe('buildLocalCapabilities / the advertised NODE_NAME frame', () => { originFloors: ORIGIN_FLOORS_CAPABILITY, tableLifecycle: TABLE_LIFECYCLE_CAPABILITY, residencyStubCopy: RESIDENCY_STUB_COPY_CAPABILITY, - residencyNotResident: 1, + residencyNotResident: RESIDENCY_NOT_RESIDENT_CAPABILITY, } ); assert.strictEqual(Object.isFrozen(local), true); @@ -287,7 +288,7 @@ describe('buildLocalCapabilities / the advertised NODE_NAME frame', () => { originFloors: ORIGIN_FLOORS_CAPABILITY, tableLifecycle: TABLE_LIFECYCLE_CAPABILITY, residencyStubCopy: RESIDENCY_STUB_COPY_CAPABILITY, - residencyNotResident: 1, + residencyNotResident: RESIDENCY_NOT_RESIDENT_CAPABILITY, } ); }); diff --git a/unitTests/replication/residencyWire.test.mjs b/unitTests/replication/residencyWire.test.mjs index a8f5e5787..c1127c47a 100644 --- a/unitTests/replication/residencyWire.test.mjs +++ b/unitTests/replication/residencyWire.test.mjs @@ -181,14 +181,14 @@ describe('residency stubs on the replication wire', function () { it('receives not-resident without a decoder or relocation and keeps absence distinct', async () => { const relocate = sinon.spy(Records, '_recordRelocate'); try { - for (const answer of [false, undefined]) { + for (const answer of [false, undefined, 0, null]) { const pending = session.getRecord({ table: Records, id: 'stub' }); const request = socket.sent.filter((frame) => frame[0] === GET_RECORD).at(-1); socket.emit( 'message', - encode(answer === false ? [GET_RECORD_RESPONSE, request[1], false] : [GET_RECORD_RESPONSE, request[1]]) + encode(answer === undefined ? [GET_RECORD_RESPONSE, request[1]] : [GET_RECORD_RESPONSE, request[1], answer]) ); - assert.equal(await pending, answer); + assert.strictEqual(await pending, answer === false ? false : undefined); } assert.equal(relocate.callCount, 0); } finally { @@ -286,7 +286,7 @@ describe('residency stubs on the replication wire', function () { const StructonPackr = createStructon(Packr); const decoder = new StructonPackr({ typedStructs: structure.typedStructs, structures: structure.structures }); assert.equal(decoder.decode(response[2].value).body, 'complete'); - } else assert.equal(response[2], id === 'stub' ? false : undefined); + } else assert.strictEqual(response[2], id === 'stub' ? false : undefined); }); } From 6b495a07bbc5274d22a73125b9b1c493fea36633 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Fri, 23 Oct 2026 08:49:10 -0600 Subject: [PATCH 08/11] chore: update core residency response contract documentation Dispatch-Task: hp-residency-not-resident-answer Co-Authored-By: GPT-5 Codex --- core | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core b/core index 7fbacfa47..572fdfcb4 160000 --- a/core +++ b/core @@ -1 +1 @@ -Subproject commit 7fbacfa472dc87d0659dbbfbb14be207754777ac +Subproject commit 572fdfcb410b5d64100c6b2d511abed4c2fb00b3 From d53b2d8e38d5ce7fd1f92f2acf3ef473c9dabb94 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Fri, 23 Oct 2026 08:49:10 -0600 Subject: [PATCH 09/11] fix: preserve versioned tombstones as terminal residency answers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Retain main’s deletion frame for all requesters. Cover deletion before an older live or unreachable holder through the sender, receiver and loader for stored and ID-derived lists. Dispatch-Task: hp-residency-not-resident-answer Co-Authored-By: GPT-5 Codex --- replication/DESIGN.md | 18 +++-- replication/replicationConnection.ts | 5 -- unitTests/replication/residencyWire.test.mjs | 69 +++++++++++++++++++- 3 files changed, 79 insertions(+), 13 deletions(-) diff --git a/replication/DESIGN.md b/replication/DESIGN.md index e3779faeb..8b5510fa6 100644 --- a/replication/DESIGN.md +++ b/replication/DESIGN.md @@ -342,12 +342,16 @@ copy/retrieval paths, except the base copy to a peer without `residencyStubCopy` unless residency is by id. The copy walk uses `invalidate` for `INVALIDATED` rows and translates the stored residency flag to the audit flag. GET_RECORD refuses stubs with `[GET_RECORD_RESPONSE, requestId, false]` for a learned requester advertising -`residencyNotResident`, and supplies current decoder structures for full rows. A missing or deleted -row keeps the two-element empty response. The receiver forwards only strict `false` as not resident, -without decoding or relocation. `Replicator.load` tries remaining available holders after either -negative answer for stored and id-derived lists. A full record wins. Otherwise absence requires -every listed non-self holder to answer and at least one genuine empty answer. Any unreachable -holder keeps the result at 502; all not-resident answers produce a 502 naming that condition. +`residencyNotResident`, and supplies current decoder structures for versioned answers. A live record +or a tombstone (null value with its deletion version) is positive evidence from a holder and settles +the read immediately. GET_RECORD preserves the versioned tombstone response for every requester. +It does not compare versions across holders or perform a quorum/read-repair read, for live records +or deletions. Only a missing row returns the two-element unversioned empty response. +The receiver forwards only strict `false` as not resident, without decoding or relocation. +`Replicator.load` tries remaining available holders after either unversioned empty or not-resident +answers for stored and id-derived lists. If no versioned answer arrives, absence requires every +listed non-self holder to answer and at least one unversioned empty answer. An unreachable holder +then keeps the result at 502; all not-resident answers produce a 502 naming that condition. The first network error remains the `ServerError` cause. Blob repair shares a subscription socket, so its request and response decoder use the peer's table id; a missing usable decoder fails locally. After a reconnect, an idle table may have no decoder until its next replicated write, so blob repair can fail for that table in the meantime. @@ -361,7 +365,7 @@ equal version; a receiver without the row stores the projection as complete — GET_RECORD empty answer stay ungated: older receivers already apply an invalidation to a non-resident, and read an empty answer as absence without storing anything. An older responder still answers a stub as empty, which the requester must count as absence. A lagging holder can also -report absence before receiving the row; this protocol has no version-qualified absence proof. +report absence before receiving the row; an unversioned empty reply supplies no deletion version. Requiring every listed holder to answer prevents an empty reply from hiding a record on an unreachable holder. It cannot prove absence outside the listed holders, or distinguish legacy stubs. GET_RECORD has no per-request timer; its wait depends on the socket watchdog. diff --git a/replication/replicationConnection.ts b/replication/replicationConnection.ts index 553c824df..166889475 100644 --- a/replication/replicationConnection.ts +++ b/replication/replicationConnection.ts @@ -5998,11 +5998,6 @@ export function replicateOverWS(ws: ReplicationWebSocket, options: any, authoriz } else { entry.version = recordVersion; } - // A stored deletion is msgpack null after its metadata prefix. - if (valueBuffer.length === 1 && valueBuffer[0] === 0xc0) { - ws.send(encode([GET_RECORD_RESPONSE, requestId])); - break; - } if (entry[METADATA] & HAS_BLOBS) { // if there are blobs, we need to find them and send their contents // but first, the decoding process can destroy our buffer above, so we need to copy it diff --git a/unitTests/replication/residencyWire.test.mjs b/unitTests/replication/residencyWire.test.mjs index c1127c47a..5c3a56812 100644 --- a/unitTests/replication/residencyWire.test.mjs +++ b/unitTests/replication/residencyWire.test.mjs @@ -207,7 +207,7 @@ describe('residency stubs on the replication wire', function () { socket.emit('message', encode([NODE_NAME, PEER, DATABASE, [], bag])); await until(() => socket.sent.some((frame) => frame[0] === NODE_NAME)); } - for (const [index, id] of ['stub', 'missing', 'deleted'].entries()) { + for (const [index, id] of ['stub', 'missing'].entries()) { socket.emit('message', encode([GET_RECORD, index, Records.tableId, id, Records.tableName])); await until(() => socket.sent.some((frame) => frame[0] === GET_RECORD_RESPONSE && frame[1] === index)); const bytes = socket.rawSent.find((frame) => { @@ -216,6 +216,11 @@ describe('residency stubs on the replication wire', function () { }); assert.ok(bytes.equals(encode([GET_RECORD_RESPONSE, index]))); } + socket.emit('message', encode([GET_RECORD, 8, Records.tableId, 'deleted', Records.tableName])); + await until(() => socket.sent.some((frame) => frame[0] === GET_RECORD_RESPONSE && frame[1] === 8)); + const tombstone = socket.sent.find((frame) => frame[0] === GET_RECORD_RESPONSE && frame[1] === 8)[2]; + assert.equal(tombstone?.version, Records.primaryStore.getEntry('deleted').version); + assert.strictEqual(decode(tombstone.value), null); const pending = session.getRecord({ table: Records, id: 'missing' }); const request = socket.sent.filter((frame) => frame[0] === GET_RECORD).at(-1); socket.emit('message', encode([GET_RECORD_RESPONSE, request[1]])); @@ -286,10 +291,72 @@ describe('residency stubs on the replication wire', function () { const StructonPackr = createStructon(Packr); const decoder = new StructonPackr({ typedStructs: structure.typedStructs, structures: structure.structures }); assert.equal(decoder.decode(response[2].value).body, 'complete'); + } else if (id === 'deleted') { + assert.equal(response[2]?.version, Records.primaryStore.getEntry('deleted').version); + assert.strictEqual(decode(response[2].value), null); } else assert.strictEqual(response[2], id === 'stub' ? false : undefined); }); } + for (const list of ['stored', 'id-derived']) { + for (const second of ['older-live', 'offline']) { + it(`a versioned deletion settles a ${list} read before an ${second} holder`, async () => { + socket.emit('message', encode([GET_RECORD, 950, Records.tableId, 'deleted', Records.tableName])); + await until(() => socket.sent.some((frame) => frame[0] === GET_RECORD_RESPONSE && frame[1] === 950)); + const wireAnswer = socket.sent.find((frame) => frame[0] === GET_RECORD_RESPONSE && frame[1] === 950)[2]; + const structure = socket.sent.find((frame) => frame[0] === TABLE_FIXED_STRUCTURE); + socket.emit('message', encode(structure)); + await new Promise((resolve) => setImmediate(resolve)); + const deletionVersion = Records.primaryStore.getEntry('deleted').version; + const older = Records.primaryStore.getEntry('full'); + assert.ok(older.version < deletionVersion); + const peers = [`deletion-${list}-${second}-first`, `deletion-${list}-${second}-second`]; + const contacted = []; + const connections = []; + const sandbox = sinon.createSandbox(); + try { + const relocate = sandbox.stub(Records, '_recordRelocate').returns(true); + sandbox.stub(getHDBNodeTable().primaryStore, 'getSync').callsFake((name) => ({ + name, + url: `wss://${name}:9933`, + })); + sandbox.stub(NodeReplicationConnection.prototype, 'connect').callsFake(function () { + connections.push(this); + this.isConnected = this.nodeName === peers[0] || second !== 'offline'; + }); + sandbox.stub(NodeReplicationConnection.prototype, 'getRecord').callsFake(async function (request) { + contacted.push(this.nodeName); + if (this.nodeName === peers[1]) + return { key: 'deleted', value: { ...older.value, id: 'deleted' }, version: older.version }; + const pending = session.getRecord(request); + const sent = socket.sent.filter((frame) => frame[0] === GET_RECORD).at(-1); + socket.emit('message', encode([GET_RECORD_RESPONSE, sent[1], wireAnswer])); + return pending; + }); + for (const [index, name] of peers.entries()) + getReplicationSharedStatus(Records.auditStore, DATABASE, name)[LATENCY_POSITION] = index; + Records.dbisDB.putSync([Symbol.for('residency_by_id'), 790], peers); + const found = await ReplicationSource.load({ + key: 'deleted', + residencyId: 790, + ...(list === 'id-derived' ? { residency: peers } : {}), + }); + assert.strictEqual(found.value, null); + assert.equal(found.version, deletionVersion); + assert.deepEqual(contacted, [peers[0]]); + assert.equal(relocate.callCount, 1); + assert.strictEqual(relocate.firstCall.args[1], found); + } finally { + for (const connection of connections) { + connection.isFinished = true; + connection.emit('finished'); + } + sandbox.restore(); + } + }); + } + } + async function copiedRecords(copySocket = socket) { copySocket.emit( 'message', From 278a3c71b1566b265b7fa99e002101d39172692b Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Fri, 23 Oct 2026 09:56:40 -0600 Subject: [PATCH 10/11] Sync core residency response documentation Dispatch-Task: hp-residency-not-resident-answer Co-Authored-By: GPT-5 Codex --- core | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core b/core index 572fdfcb4..2363d4846 160000 --- a/core +++ b/core @@ -1 +1 @@ -Subproject commit 572fdfcb410b5d64100c6b2d511abed4c2fb00b3 +Subproject commit 2363d484684268f1609b55f4e5eaae1226ab739d From dc2198966175102876e469a4a0c68134cf1b1330 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Fri, 23 Oct 2026 09:56:41 -0600 Subject: [PATCH 11/11] Clarify weak empty replies and pin residency test workers Dispatch-Task: hp-residency-not-resident-answer Co-Authored-By: GPT-5 Codex --- integrationTests/cluster/residencyReads.test.mjs | 1 + replication/DESIGN.md | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/integrationTests/cluster/residencyReads.test.mjs b/integrationTests/cluster/residencyReads.test.mjs index 2fe16fe31..ac2c9daef 100644 --- a/integrationTests/cluster/residencyReads.test.mjs +++ b/integrationTests/cluster/residencyReads.test.mjs @@ -63,6 +63,7 @@ suite('partial replication reads and placement', { timeout: 600_000 }, (ctx) => }); await startHarper(context, { config: { + threads: { count: 1 }, analytics: { aggregatePeriod: -1 }, logging: { colors: false, stdStreams: true, console: true, level: 'warn' }, replication: { securePort: hostname + ':9933' }, diff --git a/replication/DESIGN.md b/replication/DESIGN.md index 8b5510fa6..073ab73f2 100644 --- a/replication/DESIGN.md +++ b/replication/DESIGN.md @@ -346,10 +346,12 @@ stubs with `[GET_RECORD_RESPONSE, requestId, false]` for a learned requester adv or a tombstone (null value with its deletion version) is positive evidence from a holder and settles the read immediately. GET_RECORD preserves the versioned tombstone response for every requester. It does not compare versions across holders or perform a quorum/read-repair read, for live records -or deletions. Only a missing row returns the two-element unversioned empty response. +or deletions. A missing row, or a stub served to a requester without learned +`residencyNotResident` support, returns the two-element unversioned empty response. The receiver forwards only strict `false` as not resident, without decoding or relocation. `Replicator.load` tries remaining available holders after either unversioned empty or not-resident -answers for stored and id-derived lists. If no versioned answer arrives, absence requires every +answers for stored and id-derived lists. Requests remain sequential, so an id-derived miss may +wait for one GET_RECORD round trip per listed non-self holder. If no versioned answer arrives, absence requires every listed non-self holder to answer and at least one unversioned empty answer. An unreachable holder then keeps the result at 502; all not-resident answers produce a 502 naming that condition. The first network error remains the `ServerError` cause. Blob repair shares a subscription socket, so its request and response decoder