From 492163d2c5f62e55eb47ee6ac3d3cbb463709c00 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Tue, 10 Nov 2026 21:12:51 -0700 Subject: [PATCH 01/11] Validate, clear and arm table TTLs the same way on every thread A table's expiration and eviction are validated where a declaration or create_table names them, a schema that no longer declares one clears the saved value on every thread and after a restart, the cleanup scan's cadence comes from one rule for every path that sets a TTL, and under threads: 0 the scans of catalog-loaded tables arm once the main thread is worker 0, after components load. - declareTable normalizes expiration, eviction and scanInterval before anything is opened or persisted (non-negative, finite, representable numbers or numeric strings; null clears); a bad value fails with a 400 naming the table and field. create_table forwards eviction. - @table and defineTable pass null for an omitted expiration or eviction; persistTableSettings deletes a cleared field; a declaration keeps the saved value of a field it omits; a loaded class adopts a cleared catalog row; a schema without a TTL leaves one application code configured. - setTTLExpiration validates every value before changing anything; its shared cadence block falls back to the daily default when no TTL remains, instead of disabling the scan. - Saved values no declaration could save now load as absent with a warning. - describe_table reports no expiration once a TTL is cleared. Co-Authored-By: Claude Opus 5.5 Dispatch-Task: proposal-kriszyp-bundle-harper-table-expiration-config-5df9b5 --- dataLayer/DESIGN.md | 2 +- dataLayer/harperBridge/ResourceBridge.ts | 1 + dataLayer/schemaDescribe.ts | 3 +- docs/reference/database/schema.md | 6 + docs/reference/operations-api/operations.md | 2 +- .../removing-a-table-expiration-clears-it.md | 7 + .../config.yaml | 5 + .../schema.graphql | 4 + .../database/table-expiration-config.test.ts | 396 ++++++++++++++++++ .../table-expiration-config/config.yaml | 5 + .../table-expiration-config/resources.js | 47 +++ .../table-expiration-config/schema.graphql | 5 + resources/DESIGN.md | 2 +- resources/MAP.md | 2 +- resources/Table.ts | 118 ++++-- resources/databases.ts | 89 +++- resources/defineTable.ts | 5 +- resources/graphql.ts | 3 + server/threads/socketRouter.ts | 3 + .../resources/createTableCatalogOrder.test.js | 6 +- .../tableExpirationPersistence.test.js | 167 ++++++++ .../resources/ttlCleanupInterval.test.js | 182 +++++++- 22 files changed, 991 insertions(+), 69 deletions(-) create mode 100644 docs/release-notes/v5-lincoln/5.4/removing-a-table-expiration-clears-it.md create mode 100644 integrationTests/database/table-expiration-config-negative/config.yaml create mode 100644 integrationTests/database/table-expiration-config-negative/schema.graphql create mode 100644 integrationTests/database/table-expiration-config.test.ts create mode 100644 integrationTests/database/table-expiration-config/config.yaml create mode 100644 integrationTests/database/table-expiration-config/resources.js create mode 100644 integrationTests/database/table-expiration-config/schema.graphql diff --git a/dataLayer/DESIGN.md b/dataLayer/DESIGN.md index 6062e930d0..481420d215 100644 --- a/dataLayer/DESIGN.md +++ b/dataLayer/DESIGN.md @@ -42,7 +42,7 @@ Set `audit = true` on the directive's `CreateTableObject`: `createTables()` audi - A node that never runs the owner's code path never declares the rest, and its replicated rows are never removed. - An install on a pre-release of the directive's version never runs it (the `compareVersions` caveat). -Repeating a local `table()` declaration repairs any of these in place, keeping every row, and is a no-op once the shape matches. Storage-layer limits: a thread that only loads the table scans daily, so a spent row can stay a day (reads skip it); under `threads: 0` nothing scans until the first expiring write or declaration after restart; and a dropped attribute's index store stays on disk, since `table()` looks it up by table name. The certificate verification tables follow the same pattern through `security/certificateVerification/verificationTables.ts`; their rows are covered in security/DESIGN.md. +Repeating a local `table()` declaration repairs any of these in place, keeping every row, and is a no-op once the shape matches. Storage-layer limit: a dropped attribute's index store stays on disk, since `table()` looks it up by table name. The certificate verification tables follow the same pattern through `security/certificateVerification/verificationTables.ts`; their rows are covered in security/DESIGN.md. ## RocksDB restore: the restore lock + marker protocol (`dataLayer/restoreMarker.ts`) diff --git a/dataLayer/harperBridge/ResourceBridge.ts b/dataLayer/harperBridge/ResourceBridge.ts index 0dc8da27e0..d014b4e78c 100644 --- a/dataLayer/harperBridge/ResourceBridge.ts +++ b/dataLayer/harperBridge/ResourceBridge.ts @@ -168,6 +168,7 @@ export class ResourceBridge extends BridgeMethods { attributes, schemaDefined, expiration: tableCreateObj.expiration, + eviction: tableCreateObj.eviction, audit: tableCreateObj.audit, }); } diff --git a/dataLayer/schemaDescribe.ts b/dataLayer/schemaDescribe.ts index 909e94e66c..68aa06dbd5 100644 --- a/dataLayer/schemaDescribe.ts +++ b/dataLayer/schemaDescribe.ts @@ -222,7 +222,8 @@ async function descTable(describeTableObject: any, attrPerms?: any) { db_size, }; if (tableObj.replicate !== undefined) tableResult.replicate = tableObj.replicate; - if (tableObj.expirationMS !== undefined) tableResult.expiration = tableObj.expirationMS / 1000 + 's'; + // an eviction-only table reports 0s; a cleared one, like one that never had a TTL, reports none + if (tableObj.expirationMS || tableObj.evictionMS) tableResult.expiration = tableObj.expirationMS / 1000 + 's'; if (tableObj.sealed !== undefined) tableResult.sealed = tableObj.sealed; if (tableObj.cacheControl != null) tableResult.cacheControl = tableObj.cacheControl; if ((tableObj as any).sources?.length > 0) diff --git a/docs/reference/database/schema.md b/docs/reference/database/schema.md index c89f0428e3..3e93ad9bd8 100644 --- a/docs/reference/database/schema.md +++ b/docs/reference/database/schema.md @@ -107,6 +107,12 @@ type WeatherCache @table(expiration: 300, eviction: 3300, scanInterval: 600) { } ``` +**Changing or removing them** + +`expiration` and `eviction` are saved with the table, so every thread and every restart applies what the schema last declared. Removing `expiration` or `eviction` from `@table` (or from the `defineTable` options), or setting it to `0`, clears the saved value: records written afterwards no longer expire, and records already written keep the expiry they were given. Before v5.4.0 a removed value stayed in effect until the table was dropped. A TTL that application code sets at runtime (for example through `sourcedFrom` options) is not cleared by a schema that declares none. + +Each value must be a number of seconds, zero or more. A negative or non-numeric value fails the declaration (or the `create_table` operation) with an error naming the table and the argument, and the table is left unchanged. + #### How `scanInterval` Determines the Eviction Cycle `scanInterval` determines fixed clock-aligned times when eviction runs. Harper divides the clock into evenly spaced anchors based on the interval, calculated in the server's local timezone. As a result: diff --git a/docs/reference/operations-api/operations.md b/docs/reference/operations-api/operations.md index 6ba366e16f..7520668a2d 100644 --- a/docs/reference/operations-api/operations.md +++ b/docs/reference/operations-api/operations.md @@ -175,7 +175,7 @@ Drops a database and all its tables/records. **Replicates to all cluster nodes b ### `create_table` -Creates a new table. Optional fields: `database` (defaults to `data`), `attributes` (array defining schema), `expiration` (TTL in seconds). +Creates a new table. Optional fields: `database` (defaults to `data`), `attributes` (array defining schema), `expiration` (TTL in seconds), `eviction` (seconds after `expiration` before a record is removed; see [`@table`](../database/schema.md#table)). A negative or non-numeric `expiration` or `eviction` is rejected with a 400 error. ```json { diff --git a/docs/release-notes/v5-lincoln/5.4/removing-a-table-expiration-clears-it.md b/docs/release-notes/v5-lincoln/5.4/removing-a-table-expiration-clears-it.md new file mode 100644 index 0000000000..6a167980ba --- /dev/null +++ b/docs/release-notes/v5-lincoln/5.4/removing-a-table-expiration-clears-it.md @@ -0,0 +1,7 @@ +--- +section: 'Storage' +--- + +### Removing a Table's Expiration Clears It + +Removing `expiration` or `eviction` from `@table` (or from `defineTable` options), or setting it to `0`, now clears the saved value on every thread and after a restart. Before, the old value stayed in effect, so a table kept expiring records after its schema stopped declaring a TTL. If you relied on that, declare the value again. Records already written keep the expiry they were given. A negative or non-numeric `expiration`, `eviction` or `scanInterval` now fails the declaration or `create_table` with an error naming the table and argument, instead of failing the table later; `create_table` also accepts `eviction`. Expired records are now removed on schedule after a restart with `threads.count: 0`, and on a table that has only `eviction`, without waiting for a write. See [`@table`](/reference/v5/database/schema#table). diff --git a/integrationTests/database/table-expiration-config-negative/config.yaml b/integrationTests/database/table-expiration-config-negative/config.yaml new file mode 100644 index 0000000000..bb034a091d --- /dev/null +++ b/integrationTests/database/table-expiration-config-negative/config.yaml @@ -0,0 +1,5 @@ +rest: true +graphqlSchema: + files: 'schema.graphql' + # Shortened from the 30s default so a run that times out instead of failing fast costs seconds. + timeout: 5000 diff --git a/integrationTests/database/table-expiration-config-negative/schema.graphql b/integrationTests/database/table-expiration-config-negative/schema.graphql new file mode 100644 index 0000000000..7dcc9649cc --- /dev/null +++ b/integrationTests/database/table-expiration-config-negative/schema.graphql @@ -0,0 +1,4 @@ +type NegativeTtl @table(database: "ttl_config_bad", expiration: -1) @export { + id: ID @primaryKey + value: String +} diff --git a/integrationTests/database/table-expiration-config.test.ts b/integrationTests/database/table-expiration-config.test.ts new file mode 100644 index 0000000000..b79948dcdf --- /dev/null +++ b/integrationTests/database/table-expiration-config.test.ts @@ -0,0 +1,396 @@ +/** + * A table's TTL configuration on every thread that holds the table: the threads that declare it, the threads + * that only load it from the catalog, and the main thread under `threads: 0`, across restarts, on RocksDB and + * LMDB (harper#1286, #2831, #3372, #3934, #3935). + * + * The cleanup scan runs on the store-maintenance owner, the last worker. A table created by `create_table`, or + * declared through the fixture's TtlControl endpoint on another worker, reaches the owner only through the + * catalog. Under `threads: 0` the main thread loads every table before it becomes worker 0. + * + * Expiration and eviction are 2 s each (8 s on the tables that live across a restart), so the derived scan + * interval is (2 + 2) / 4 = 1 s (4 s). + * + * Reproduction: + * npm run build && npm run test:integration -- "integrationTests/database/table-expiration-config.test.ts" + */ +import { suite, test, before, after } from 'node:test'; +import { ok, strictEqual } from 'node:assert'; +import { request as httpRequest } from 'node:http'; +import { join, resolve } from 'node:path'; +import { cp, mkdtemp, readFile, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { setTimeout as sleep } from 'node:timers/promises'; +import { startHarper, killHarper, teardownHarper, type ContextWithHarper } from '@harperfast/integration-testing'; + +const FIXTURE = 'table-expiration-config'; +const NEGATIVE_FIXTURE = 'table-expiration-config-negative'; +const DATABASE = 'ttl_config'; +const EXPIRATION = 2; +const EVICTION = 2; +const ROWS = 5; +// Tables that live across a restart use a longer TTL, so their rows only become evictable after the restart and +// only the restarted process can reclaim them. Derived scan interval (8 + 8) / 4 = 4 s. +const RUNTIME_EXPIRATION = 8; +const RUNTIME_EVICTION = 8; +const RECLAIM_MARGIN_MS = 15_000; +// past expiration + eviction plus three scans, so a TTL still in effect would have hidden and reclaimed the rows +const CLEARED_HOLD_MS = (EXPIRATION + EVICTION) * 1000 + 3_000; + +interface OperationResult { + status: number; + body: any; +} + +async function operation(ctx: ContextWithHarper, body: Record): Promise { + const response = await fetch(ctx.harper.operationsAPIURL, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Authorization': basicAuth(ctx), + }, + body: JSON.stringify(body), + }); + const text = await response.text(); + let parsed: any = text; + try { + parsed = JSON.parse(text); + } catch { + // not JSON + } + return { status: response.status, body: parsed }; +} + +async function ok200(ctx: ContextWithHarper, body: Record) { + const result = await operation(ctx, body); + strictEqual(result.status, 200, `${body.operation} failed: ${JSON.stringify(result.body)}`); + return result.body; +} + +// describe_table counts stored entries without a query or GET, so it neither hides nor touches expired rows +async function storedRows(ctx: ContextWithHarper, table: string): Promise { + return (await ok200(ctx, { operation: 'describe_table', database: DATABASE, table })).record_count; +} + +// Polls past the moment the rows become evictable (expiry + eviction) by a few scans of the runtime cadence. +async function waitForReclaim(ctx: ContextWithHarper, table: string, dueAt: number, why: string) { + const deadline = dueAt + RECLAIM_MARGIN_MS; + let count: number; + do { + await sleep(500); + count = await storedRows(ctx, table); + } while (count > 0 && Date.now() < deadline); + strictEqual( + count, + 0, + `${count} rows of ${table} still stored ${RECLAIM_MARGIN_MS / 1000}s after they were due: ${why}` + ); +} + +async function insertRows(ctx: ContextWithHarper, table: string) { + const records = Array.from({ length: ROWS }, (_, i) => ({ id: String(i), value: `v${i}` })); + await ok200(ctx, { operation: 'insert', database: DATABASE, table, records }); +} + +const basicAuth = (ctx: ContextWithHarper) => + `Basic ${Buffer.from(`${ctx.harper.admin.username}:${ctx.harper.admin.password}`).toString('base64')}`; + +// a fresh connection per request, so retries reach different workers +function http( + ctx: ContextWithHarper, + method: string, + path: string, + body?: Record +): Promise<{ status: number; text: string }> { + const payload = body === undefined ? undefined : JSON.stringify(body); + return new Promise((resolveRequest, reject) => { + const req = httpRequest( + new URL(path, ctx.harper.httpURL), + { + method, + agent: false, + headers: { + Authorization: basicAuth(ctx), + ...(payload === undefined + ? {} + : { 'Content-Type': 'application/json', 'Content-Length': Buffer.byteLength(payload) }), + }, + }, + (res) => { + const chunks: Buffer[] = []; + res.on('data', (chunk) => chunks.push(chunk)); + res.on('end', () => resolveRequest({ status: res.statusCode ?? 0, text: Buffer.concat(chunks).toString() })); + } + ); + req.setTimeout(30_000, () => req.destroy(new Error(`${method} ${path}: no response`))); + req.on('error', reject); + req.end(payload); + }); +} + +async function control(ctx: ContextWithHarper, body: Record): Promise { + const { status, text } = await http(ctx, 'POST', '/TtlControl/', body); + if (status !== 200) throw new Error(`TtlControl ${body.op}: ${status} ${text}`); + return JSON.parse(text); +} + +async function untilServed(ctx: ContextWithHarper, body: Record) { + const deadline = Date.now() + 60_000; + for (;;) { + const result = await control(ctx, body); + if (!result.wrongThread) return result; + if (Date.now() > deadline) throw new Error(`TtlControl ${body.op} never reached a worker that could serve it`); + await sleep(10); + } +} + +const workerTtl = (ctx: ContextWithHarper, table: string, worker: number) => + untilServed(ctx, { op: 'ttl', table, onWorker: worker }); + +async function waitForWorkerTtl( + ctx: ContextWithHarper, + table: string, + worker: number, + expected: { expirationMS: number; evictionMS: number } +) { + const deadline = Date.now() + 20_000; + let seen: any; + do { + seen = await workerTtl(ctx, table, worker); + if (seen.loaded && seen.expirationMS === expected.expirationMS && seen.evictionMS === expected.evictionMS) return; + await sleep(200); + } while (Date.now() < deadline); + throw new Error(`worker ${worker} holds ${JSON.stringify(seen)} for ${table}, expected ${JSON.stringify(expected)}`); +} + +async function assertRowsKept(ctx: ContextWithHarper, table: string, why: string) { + await sleep(CLEARED_HOLD_MS); + strictEqual(await storedRows(ctx, table), ROWS, `rows of ${table} were reclaimed on: ${why}`); + for (let i = 0; i < ROWS * 4; i++) { + const { status } = await http(ctx, 'GET', `/${table}/${i % ROWS}`); + strictEqual(status, 200, `GET /${table}/${i % ROWS} was hidden as expired on: ${why}`); + } +} + +// Every check after a restart runs, so one run on a base that lacks a fix reports each check it fails. +function softChecks() { + const failures: string[] = []; + return { + async check(run: () => Promise) { + try { + await run(); + } catch (error) { + failures.push((error as Error).message); + } + }, + assertNone() { + strictEqual(failures.length, 0, `${failures.length} checks failed:\n- ${failures.join('\n- ')}`); + }, + }; +} + +async function stage(fixtures: string[]) { + const dataRootDir = await mkdtemp( + join(process.env.HARPER_INTEGRATION_TEST_INSTALL_PARENT_DIR || tmpdir(), 'harper-integration-test-') + ); + for (const fixture of fixtures) + await cp(resolve(import.meta.dirname, fixture), join(dataRootDir, 'components', fixture), { + recursive: true, + dereference: true, + }); + return dataRootDir; +} + +async function dropDeclaredTtl(ctx: ContextWithHarper) { + const schemaPath = join(ctx.harper.dataRootDir, 'components', FIXTURE, 'schema.graphql'); + const schema = await readFile(schemaPath, 'utf8'); + const cleared = schema.replace(`, expiration: ${EXPIRATION}, eviction: ${EVICTION}`, ''); + ok(cleared !== schema, 'the fixture schema no longer declares the TTL this test removes'); + await writeFile(schemaPath, cleared); +} + +async function createRuntimeTtl(ctx: ContextWithHarper) { + await ok200(ctx, { + operation: 'create_table', + database: DATABASE, + table: 'RuntimeTtl', + primary_key: 'id', + expiration: RUNTIME_EXPIRATION, + eviction: RUNTIME_EVICTION, + }); +} + +// The restarted process loads every table from the catalog; the fixture schema re-declares Declared without a TTL. +async function restartWithoutDeclaredTtl(ctx: ContextWithHarper, options: Record, dueAt: number) { + await killHarper(ctx); + await dropDeclaredTtl(ctx); + await startHarper(ctx, options); + ok(Date.now() < dueAt, 'the restart outlasted the runtime TTL, so it cannot show which process reclaimed the rows'); +} + +async function assertDeclaredTtlRemoved(ctx: ContextWithHarper) { + const described = await ok200(ctx, { operation: 'describe_table', database: DATABASE, table: 'Declared' }); + strictEqual(described.expiration, undefined, 'describe_table (the main thread) still reports the removed expiration'); + await insertRows(ctx, 'Declared'); + await assertRowsKept(ctx, 'Declared', 'a schema whose TTL was removed, after a restart'); +} + +for (const engine of ['rocksdb', 'lmdb']) { + const env = engine === 'lmdb' ? { HARPER_STORAGE_ENGINE: 'lmdb' } : {}; + + suite(`Table TTL configuration with four workers (${engine})`, (ctx: ContextWithHarper) => { + const THREADS = 4; + const OWNER = THREADS - 1; + const options = { config: { threads: { count: THREADS } }, env }; + + before(async () => { + ctx.harper = { dataRootDir: await stage([FIXTURE, NEGATIVE_FIXTURE]) } as any; + await startHarper(ctx, options); + }); + + after(async () => { + await teardownHarper(ctx); + }); + + test('a negative TTL fails where it is declared, naming the table and field (#1286, #3935)', async () => { + const name = `${NEGATIVE_FIXTURE}.graphqlSchema`; + const deadline = Date.now() + 30_000; + let entry: any; + do { + const status = await ok200(ctx, { operation: 'get_status' }); + entry = status.componentStatus?.find((candidate: any) => candidate.name === name); + if (entry?.status === 'error') break; + await sleep(250); + } while (Date.now() < deadline); + strictEqual( + entry?.status, + 'error', + `@table(expiration: -1) did not fail its component: ${JSON.stringify(entry)}` + ); + const reported = JSON.stringify(entry); + ok(!/timed out/i.test(reported), `the component failed on a timeout instead of the TTL: ${reported}`); + ok( + /expiration/.test(reported) && reported.includes('ttl_config_bad.NegativeTtl'), + `the failure does not name the table and field: ${reported}` + ); + + for (const [field, table] of [ + ['eviction', 'NegativeEviction'], + ['expiration', 'NegativeExpiration'], + ]) { + const created = await operation(ctx, { + operation: 'create_table', + database: DATABASE, + table, + primary_key: 'id', + [field]: -1, + }); + strictEqual(created.status, 400, `create_table with ${field}: -1 returned ${JSON.stringify(created)}`); + const error = JSON.stringify(created.body); + ok( + error.includes(field) && error.includes(`${DATABASE}.${table}`), + `create_table does not name the table and field: ${error}` + ); + const described = await operation(ctx, { operation: 'describe_table', database: DATABASE, table }); + ok(described.status !== 200, `a rejected create_table left ${table} behind`); + } + }); + + test('the declaring thread reclaims expired rows on the derived cadence', { timeout: 60_000 }, async () => { + await insertRows(ctx, 'Declared'); + await waitForReclaim(ctx, 'Declared', Date.now() + (EXPIRATION + EVICTION) * 1000, 'a schema-declared TTL'); + }); + + test('an eviction-only TTL adopted from the catalog reclaims rows (#3935)', { timeout: 120_000 }, async () => { + const table = 'AdoptedEviction'; + const { workerIndex } = await untilServed(ctx, { op: 'declare', table, avoidWorker: OWNER }); + // loaded before it gains a TTL, so the owner adopts the TTL instead of loading the table fresh with it + await waitForWorkerTtl(ctx, table, OWNER, { expirationMS: 0, evictionMS: 0 }); + await untilServed(ctx, { op: 'declare', table, eviction: EVICTION, onWorker: workerIndex }); + await waitForWorkerTtl(ctx, table, OWNER, { expirationMS: 0, evictionMS: EVICTION * 1000 }); + await untilServed(ctx, { op: 'write', table, count: ROWS, expiresInMs: 500 }); + await waitForReclaim(ctx, table, Date.now() + 500 + EVICTION * 1000, 'an eviction-only TTL the owner adopted'); + }); + + // both declarations on one worker: a worker keeps a TTL its own declaration set, as a restart re-applies it + test('a re-declaration without a TTL clears it on every worker (#3934)', { timeout: 120_000 }, async () => { + const table = 'AdoptedClear'; + const declared = { expirationMS: EXPIRATION * 1000, evictionMS: EVICTION * 1000 }; + const { workerIndex } = await untilServed(ctx, { + op: 'declare', + table, + expiration: EXPIRATION, + eviction: EVICTION, + avoidWorker: OWNER, + }); + for (let worker = 0; worker < THREADS; worker++) await waitForWorkerTtl(ctx, table, worker, declared); + await untilServed(ctx, { op: 'declare', table, onWorker: workerIndex }); + const checks = softChecks(); + for (let worker = 0; worker < THREADS; worker++) + await checks.check(() => waitForWorkerTtl(ctx, table, worker, { expirationMS: 0, evictionMS: 0 })); + checks.assertNone(); + }); + + test( + 'after a restart: a runtime table is reclaimed without a write, and a removed @table TTL stays removed (#3372, #3934)', + { timeout: 180_000 }, + async () => { + const checks = softChecks(); + await createRuntimeTtl(ctx); + await insertRows(ctx, 'RuntimeTtl'); + const dueAt = Date.now() + (RUNTIME_EXPIRATION + RUNTIME_EVICTION) * 1000; + await restartWithoutDeclaredTtl(ctx, options, dueAt); + + await checks.check(() => waitForReclaim(ctx, 'RuntimeTtl', dueAt, 'a runtime-created table, no write')); + for (let worker = 0; worker < THREADS; worker++) + await checks.check(() => waitForWorkerTtl(ctx, 'Declared', worker, { expirationMS: 0, evictionMS: 0 })); + await checks.check(() => assertDeclaredTtlRemoved(ctx)); + checks.assertNone(); + } + ); + }); + + suite(`Table TTL configuration with threads: 0 (${engine})`, (ctx: ContextWithHarper) => { + const options = { config: { threads: { count: 0 } }, env }; + + before(async () => { + ctx.harper = { dataRootDir: await stage([FIXTURE]) } as any; + await startHarper(ctx, options); + }); + + after(async () => { + await teardownHarper(ctx); + }); + + test( + 'after a restart, the main thread reclaims catalog-loaded tables without a write and keeps a removed TTL removed (#2831, #3934, #3935)', + { timeout: 180_000 }, + async () => { + const checks = softChecks(); + strictEqual((await ok200(ctx, { operation: 'get_configuration' })).threads.count, 0); + await createRuntimeTtl(ctx); + await ok200(ctx, { + operation: 'create_table', + database: DATABASE, + table: 'RuntimeEviction', + primary_key: 'id', + eviction: RUNTIME_EVICTION, + }); + await checks.check(() => + waitForWorkerTtl(ctx, 'RuntimeEviction', 0, { expirationMS: 0, evictionMS: RUNTIME_EVICTION * 1000 }) + ); + // last before the restart, so the rows only become evictable after it + await insertRows(ctx, 'RuntimeTtl'); + const expiresInMs = RUNTIME_EXPIRATION * 1000; + await untilServed(ctx, { op: 'write', table: 'RuntimeEviction', count: ROWS, expiresInMs }); + const dueAt = Date.now() + (RUNTIME_EXPIRATION + RUNTIME_EVICTION) * 1000; + await restartWithoutDeclaredTtl(ctx, options, dueAt); + + await checks.check(() => waitForReclaim(ctx, 'RuntimeTtl', dueAt, 'a runtime-created table, no write')); + await checks.check(() => waitForReclaim(ctx, 'RuntimeEviction', dueAt, 'an eviction-only runtime table')); + await checks.check(() => waitForWorkerTtl(ctx, 'Declared', 0, { expirationMS: 0, evictionMS: 0 })); + await checks.check(() => assertDeclaredTtlRemoved(ctx)); + checks.assertNone(); + } + ); + }); +} diff --git a/integrationTests/database/table-expiration-config/config.yaml b/integrationTests/database/table-expiration-config/config.yaml new file mode 100644 index 0000000000..41005f90af --- /dev/null +++ b/integrationTests/database/table-expiration-config/config.yaml @@ -0,0 +1,5 @@ +rest: true +graphqlSchema: + files: 'schema.graphql' +jsResource: + files: 'resources.js' diff --git a/integrationTests/database/table-expiration-config/resources.js b/integrationTests/database/table-expiration-config/resources.js new file mode 100644 index 0000000000..479eeaf253 --- /dev/null +++ b/integrationTests/database/table-expiration-config/resources.js @@ -0,0 +1,47 @@ +// POST /TtlControl/ drives table-expiration-config.test.ts: +// { op: 'declare', table, expiration?, eviction?, avoidWorker?, onWorker? } — defineTable on the worker that +// serves the request; a worker that avoidWorker excludes, or onWorker does not name, answers { wrongThread } +// and the test retries on a fresh connection, so the scan owner only ever learns the TTL from the catalog +// { op: 'ttl', table, onWorker } — that worker's view of the table: whether it is loaded, and its TTL in ms +// { op: 'write', table, count, expiresInMs } — rows whose own expiry is expiresInMs from now +import { Resource, databases, defineTable, types } from 'harper'; +import { workerData } from 'node:worker_threads'; + +const DATABASE = 'ttl_config'; +const workerIndex = () => workerData?.workerIndex ?? 0; + +export class TtlControl extends Resource { + static loadAsInstance = false; + async post(_target, body) { + const { op, table } = body; + if (op === 'declare') { + if ( + (body.avoidWorker !== undefined && workerIndex() === body.avoidWorker) || + (body.onWorker !== undefined && workerIndex() !== body.onWorker) + ) + return { wrongThread: true, workerIndex: workerIndex() }; + const options = { database: DATABASE }; + if (body.expiration !== undefined) options.expiration = body.expiration; + if (body.eviction !== undefined) options.eviction = body.eviction; + defineTable(table, { id: types.id.primaryKey, value: types.string.nullable }, options); + return { workerIndex: workerIndex() }; + } + if (op === 'ttl') { + if (workerIndex() !== body.onWorker) return { wrongThread: true, workerIndex: workerIndex() }; + const Table = databases[DATABASE]?.[table]; + return { + workerIndex: workerIndex(), + loaded: Boolean(Table), + expirationMS: Table?.expirationMS || 0, + evictionMS: Table?.evictionMS || 0, + }; + } + if (op === 'write') { + const Table = databases[DATABASE][table]; + const expiresAt = Date.now() + body.expiresInMs; + for (let i = 0; i < body.count; i++) await Table.put(String(i), { id: String(i), value: 'v' }, { expiresAt }); + return { workerIndex: workerIndex(), expiresAt }; + } + throw new Error(`unknown op ${op}`); + } +} diff --git a/integrationTests/database/table-expiration-config/schema.graphql b/integrationTests/database/table-expiration-config/schema.graphql new file mode 100644 index 0000000000..a0347c5115 --- /dev/null +++ b/integrationTests/database/table-expiration-config/schema.graphql @@ -0,0 +1,5 @@ +# The test rewrites this file between restarts to drop the TTL arguments. +type Declared @table(database: "ttl_config", expiration: 2, eviction: 2) @export { + id: ID @primaryKey + value: String +} diff --git a/resources/DESIGN.md b/resources/DESIGN.md index 78e0a7f892..d746c0b529 100644 --- a/resources/DESIGN.md +++ b/resources/DESIGN.md @@ -269,7 +269,7 @@ If this same write (origin and log key) emptied the key, `_writeDelete` writes n ## Cluster-origin table definitions are additive-only -An `origin: 'cluster'` definition is a peer snapshot, so `table()` only adds from it and logs what it discards. Drops and redefinitions never converge via gossip (recover via local `create_attribute` or redeploy). Abandoned index builds still recover (`clusterSchemaMerge.test.js`). A reload keeps a thread's own TTL. +An `origin: 'cluster'` definition is a peer snapshot, so `table()` only adds from it and logs what it discards. Drops and redefinitions never converge via gossip (recover via local `create_attribute` or redeploy). Abandoned index builds still recover (`clusterSchemaMerge.test.js`). A reload keeps a thread's own TTL. `@table` omitting a TTL clears it (`null`); `table()` omitting one keeps it. ## `drop_attribute` never removes the primary key, and a strip keeps each row's version diff --git a/resources/MAP.md b/resources/MAP.md index 4f2cf90865..5533f135af 100644 --- a/resources/MAP.md +++ b/resources/MAP.md @@ -46,7 +46,7 @@ Where things are in `resources/`. The rules live in [DESIGN.md](DESIGN.md): read - response shape (select) → `transformEntryForSelect` in Table.ts; `transformForSelect` in Resource.ts - selects answered from index entries alone → `coveredSelectAttribute` in Table.ts; `coverIndexEntries` in search.ts - sorting → `transformToOrderedSelect` in Table.ts; whether the lead already supplies the order → `scansInAttributeOrder` in search.ts -- record TTL and eviction → `setTTLExpiration`, `_adoptCatalogExpiration`, `evict`, `scheduleCleanup` in Table.ts +- record TTL and eviction → `setTTLExpiration`, `_adoptCatalogExpiration`, `evict`, `scheduleCleanup` in Table.ts; `armTableCleanupScans` in databases.ts - cache miss to source → `getFromSource` in Table.ts - subscriptions and replay → `subscribe`, `eventFromAudit` in Table.ts; `addSubscription` in transactionBroadcast.ts - replication apply loop → `sourcedFrom` in Table.ts diff --git a/resources/Table.ts b/resources/Table.ts index 61de4089e7..f02a6d0476 100644 --- a/resources/Table.ts +++ b/resources/Table.ts @@ -811,6 +811,34 @@ export function replicaExpirationMsOf(seconds: unknown): number | undefined { if (!Number.isFinite(ms)) throw new Error('replicaExpiration is too large'); return ms; } +// the scan cadence of a table with no TTL, which an expiring write arms for its per-record expiry +const DEFAULT_CLEANUP_INTERVAL = 86_400_000; +// the latest Date, in seconds: an expiry past it cannot be stamped on a record +const MAX_TTL_SECONDS = 8.64e12; +/** + * A table's `expiration`, `eviction` or `scanInterval` in seconds, as a declaration or `setTTLExpiration` gives + * it: undefined when not given, 0 for `null` (cleared), else a number or a numeric string (a create persisted it + * as given) that is finite and zero or more. + */ +export function ttlSecondsOf(table: string, field: string, value: unknown): number | undefined { + if (value === undefined) return undefined; + if (value === null) return 0; + const seconds = + typeof value === 'number' ? value : typeof value === 'string' && value.trim() !== '' ? Number(value) : NaN; + if (seconds >= 0 && seconds <= MAX_TTL_SECONDS) return seconds; + const shown = + typeof value === 'number' + ? String(value) + : typeof value === 'string' + ? JSON.stringify(value.slice(0, 40)) + : typeof value; + throw new ClientError(`${field} of '${table}' must be a non-negative number of seconds, got ${shown}`); +} +/** A TTL field as the catalog holds it: absent unless a valid positive number of seconds. */ +export function savedTtlSeconds(value: unknown): number | undefined { + const seconds = Number(value); + return seconds > 0 && seconds <= MAX_TTL_SECONDS ? seconds : undefined; +} const TEST_WRITE_KEY_BUFFER = Buffer.allocUnsafeSlow(8192); const MAX_KEY_BYTES = 1978; const EVENT_HIGH_WATER_MARK = 100; @@ -1196,6 +1224,7 @@ interface TableResourceClass { readonly residency: Readonly | undefined; _setResidencyPolicy(policy: ResidencyPolicy): void; _adoptCatalogExpiration(expiration?: number, eviction?: number): void; + _armCleanupScan(): void; getResidencySlot(id: Id, record?: object): number; getRecordResidency(entry: Partial): readonly string[] | undefined; get expirationMS(): any; @@ -1964,7 +1993,6 @@ export function makeTable(options): TableResourceClass { // a store on any thread wakes the sweep when [1] is 0 let replicaSweepState: Int32Array | undefined; let recordScanInterval = 0; - let recordCleanupInterval = 86400000; const replicaRefreshInFlight = new Set(); const replicaRefreshInFlightComposite = new Set(); // Set when the TTL exists only on this thread: either application code configured it at runtime, or @@ -2036,7 +2064,7 @@ export function makeTable(options): TableResourceClass { let prefetchCallbacks = []; let untilNextPrefetch = 1; let nonPrefetchSequence = 2; - let cleanupInterval = 86400000; + let cleanupInterval = DEFAULT_CLEANUP_INTERVAL; let cleanupPriority = 0; let lastCleanupInterval: number | undefined; let cleanupTimer: NodeJS.Timeout | undefined; @@ -2772,22 +2800,23 @@ export function makeTable(options): TableResourceClass { */ static _adoptCatalogExpiration(expiration?: number, eviction?: number) { if (ttlConfiguredByApplication || ttlDeclaredBySchema) return; - const loadedExpirationMs = expiration && expiration > 0 ? expiration * 1000 : 0; - const loadedEvictionMs = eviction && eviction > 0 ? eviction * 1000 : 0; - if (!loadedExpirationMs && !loadedEvictionMs) return; - if (loadedExpirationMs === expirationMs && loadedEvictionMs === evictionMs) return; - evictionMs = loadedEvictionMs; - if (!loadedExpirationMs) { - expirationMs = 0; - return; - } + expiration ||= 0; + eviction ||= 0; + if (expiration * 1000 === (expirationMs || 0) && eviction * 1000 === evictionMs) return; ttlFromLoad = true; try { - TableResource.setTTLExpiration(loadedExpirationMs / 1000); + TableResource.setTTLExpiration({ expiration, eviction }); } finally { ttlFromLoad = false; } } + /** + * Arm the cleanup scan of a table this thread loaded before it could own the scan: under `threads: 0` the + * main thread loads every table before it becomes worker 0. + */ + static _armCleanupScan() { + if (expirationScanScheduled || evictionMs) scheduleCleanup(); + } static getResidencySlot(id: Id, record?: object): number { return calculateResidencySlot(id, record); } @@ -3913,9 +3942,11 @@ export function makeTable(options): TableResourceClass { * @param opts Time in seconds until records expire, or an options object with `expiration`, `eviction`, * and `scanInterval` (all in seconds, all optional). Number form preserves any previously configured * eviction/scanInterval; object form replaces all three, unless it names only the replica options. An internal schema ownership-only call with - * none of those values preserves the settings already loaded from the catalog. + * none of those values preserves the settings already loaded from the catalog, and so does an internal + * schema call with no TTL on a thread whose application code configured one. * Either form then scans for expired records every `scanInterval`, or a quarter of expiration + eviction - * when none is configured. + * when none is configured, or daily (for per-record expiry) when the table has no TTL. A rejected value + * changes nothing. */ static setTTLExpiration( opts: @@ -3932,42 +3963,47 @@ export function makeTable(options): TableResourceClass { ) { if (opts == null || (typeof opts !== 'number' && typeof opts !== 'object')) throw new Error('Invalid expiration value type'); + const qualifiedName = `${databaseName}.${tableName}`; + const byNumber = typeof opts === 'number'; + const expirationSeconds = ttlSecondsOf(qualifiedName, 'expiration', byNumber ? opts : opts.expiration); + const evictionSeconds = byNumber ? undefined : ttlSecondsOf(qualifiedName, 'eviction', opts.eviction); + const scanIntervalSeconds = byNumber ? undefined : ttlSecondsOf(qualifiedName, 'scanInterval', opts.scanInterval); + const validatedReplicaExpirationMs = byNumber ? undefined : replicaExpirationMsOf(opts.replicaExpiration); + const validatedRefreshFraction = byNumber ? undefined : replicaRefreshFractionOf(opts.replicaRefreshFraction); const previousReplicaExpirationMs = replicaExpirationMs; const ownedCleanup = ownsCleanup(); - if (typeof opts === 'object' && opts.replicaExpiration != null) { - const validated = replicaExpirationMsOf(opts.replicaExpiration); - replicaExpirationMs = isRocksDB ? validated : 0; - } - if (typeof opts === 'object' && opts.replicaRefreshFraction != null) - replicaRefreshFraction = replicaRefreshFractionOf(opts.replicaRefreshFraction); - const declaredHere = typeof opts === 'object' && opts.fromSchema; + if (validatedReplicaExpirationMs !== undefined) + replicaExpirationMs = isRocksDB ? validatedReplicaExpirationMs : 0; + if (validatedRefreshFraction !== undefined) replicaRefreshFraction = validatedRefreshFraction; + const declaredHere = !byNumber && opts.fromSchema; const isolatedApplicationOwner = declaredHere && opts.isolatedApplicationOwner; const preserveLoadedConfiguration = - typeof opts === 'object' && + !byNumber && (declaredHere || opts.replicaExpiration !== undefined || opts.replicaRefreshFraction !== undefined) && - opts.expiration === undefined && - opts.eviction === undefined && - opts.scanInterval === undefined; + ((opts.expiration === undefined && opts.eviction === undefined && opts.scanInterval === undefined) || + // a schema without a TTL does not clear one this thread's application code configured + (declaredHere && + ttlConfiguredByApplication && + !isolatedApplicationOwner && + !expirationSeconds && + !evictionSeconds && + !scanIntervalSeconds)); if (((!ttlFromLoad && !declaredHere) || isolatedApplicationOwner) && !ttlConfiguredByApplication) { ttlConfiguredByApplication = true; // the scan owner may have changed with this: re-evaluate even if the interval did not lastCleanupInterval = undefined; } - if (typeof opts === 'number') { - expirationMs = opts * 1000; + if (byNumber) { + expirationMs = expirationSeconds * 1000; } else if (!preserveLoadedConfiguration) { - // `??` so an explicit 0 is treated as the user's chosen value, not as "missing" - expirationMs = (opts.expiration ?? 0) * 1000; - evictionMs = (opts.eviction ?? 0) * 1000; - recordScanInterval = (opts.scanInterval ?? 0) * 1000; + expirationMs = (expirationSeconds ?? 0) * 1000; + evictionMs = (evictionSeconds ?? 0) * 1000; + recordScanInterval = (scanIntervalSeconds ?? 0) * 1000; } - if (expirationMs < 0) throw new Error('Expiration can not be negative'); if (declaredHere && !preserveLoadedConfiguration) ttlDeclaredBySchema = true; if (!preserveLoadedConfiguration) { - // default to one quarter of the total expiration+eviction window - recordCleanupInterval = recordScanInterval || (expirationMs + evictionMs) / 4; - cleanupInterval = recordCleanupInterval; - expirationScanScheduled = true; + expirationScanScheduled = Boolean(recordScanInterval || expirationMs || evictionMs); + cleanupInterval = recordScanInterval || (expirationMs + evictionMs) / 4 || DEFAULT_CLEANUP_INTERVAL; } if ( replicaTableKnown && @@ -3977,8 +4013,9 @@ export function makeTable(options): TableResourceClass { if (replicaExpirationMs) armReplicaEviction(); } else if (replicaEvictionArmed) armReplicaEviction(); // Re-evaluate an existing table-level scan after an ownership-only declaration, but do not - // create the default daily cleanup timer for a table that has only an @expiresAt field. - if (!preserveLoadedConfiguration || expirationScanScheduled || evictionMs) scheduleCleanup(); + // create the default daily cleanup timer for a table that has only an @expiresAt field. A + // cleared TTL moves an armed scan to the daily default rather than leaving its old cadence. + if (expirationScanScheduled || evictionMs || (!preserveLoadedConfiguration && cleanupTimer)) scheduleCleanup(); // @expiresAt has its own interval rather than the cleanup timer above. Arm it whenever a live // declaration introduces the attribute, including after this application already claimed TTL. if (expiresAtProperty && !recordExpirationInterval) runRecordExpirationEviction(); @@ -14424,11 +14461,6 @@ export function makeTable(options): TableResourceClass { if (ownsCleanup()) { // run on the last thread so we aren't overloading lower-numbered threads if (cleanupTimer) clearTimeout(cleanupTimer); - if (!cleanupInterval) { - // no replacement pass is being scheduled, so nothing is left to settle a superseded one - settlePendingCleanup(); - return; - } // This pass adopts the awaiters of the pass whose timer it just cleared: they settle when // this pass's scan completes, so a reclamation run is never told the storage was reclaimed // before any scan ran. It has to run now, though — that run blocks its whole path on the diff --git a/resources/databases.ts b/resources/databases.ts index 466462801c..efde371cf7 100644 --- a/resources/databases.ts +++ b/resources/databases.ts @@ -44,6 +44,8 @@ import { makeTable, replicaExpirationMsOf, replicaRefreshFractionOf, + savedTtlSeconds, + ttlSecondsOf, ignoreAlreadyDropped, acquireUpdateAttributesLock, releaseUpdateAttributesLock, @@ -1137,6 +1139,27 @@ export function catalogCreatedTime(table: { dbisDB: any; tableName: string; prim const createdTime = primaryCatalogRowFor(table)?.value?.createdTime; return Number.isFinite(createdTime) ? createdTime : undefined; } +/** Whether a declared TTL field (undefined = not declared, 0 = cleared) differs from the catalog's. */ +function ttlChanged(declared: number | undefined, saved: unknown): boolean { + if (declared === undefined) return false; + return declared ? Number(saved) !== declared : saved !== undefined; +} +const warnedInvalidSavedTtl = new Set(); +/** + * A saved TTL field as a load applies it: a value no declaration could save now (a negative eviction or a + * non-numeric expiration an older create persisted) loads as absent, and the next declaration rewrites it. + */ +function loadedTtlSeconds(databaseName: string, tableName: string, field: string, saved: unknown) { + const seconds = savedTtlSeconds(saved); + const key = `${databaseName}.${tableName}.${field}`; + if (seconds === undefined && saved !== undefined && saved !== 0 && !warnedInvalidSavedTtl.has(key)) { + warnedInvalidSavedTtl.add(key); + logger.warn( + `Ignoring the saved ${field} ${JSON.stringify(saved)} of ${databaseName}.${tableName}: not a valid TTL` + ); + } + return seconds; +} /** * The table-level expiration and eviction (seconds) on disk, each present only when set. A class on a thread * that application code configured can hold a TTL the catalog does not. @@ -1148,14 +1171,25 @@ export function catalogExpiration(table: { dbisDB: any; tableName: string; prima // the row declareTable writes the settings to and initStores loads them from: a named primary row wins const named = table.primaryKey ? table.dbisDB?.getSync(`${table.tableName}/${table.primaryKey}`) : undefined; const row = named?.isPrimaryKey ? named : table.dbisDB?.getSync(`${table.tableName}/`); - // a create persists the declared value as given, which can be a numeric string - const expiration = Number(row?.expiration); - const eviction = Number(row?.eviction); + // a create persisted the declared value as given, which can be a numeric string + const expiration = savedTtlSeconds(row?.expiration); + const eviction = savedTtlSeconds(row?.eviction); const settings: { expiration?: number; eviction?: number } = {}; - if (expiration > 0 && Number.isFinite(expiration)) settings.expiration = expiration; - if (eviction > 0 && Number.isFinite(eviction)) settings.eviction = eviction; + if (expiration) settings.expiration = expiration; + if (eviction) settings.eviction = eviction; return settings; } +/** + * Arms the cleanup scan of every loaded table that has a TTL. Under `threads: 0` the main thread loads every + * table before it becomes worker 0, when no thread can own a scan yet. + */ +export function armTableCleanupScans(): void { + const loaded = getDatabases(); + // system is not enumerable + for (const database of new Set([loaded.system, ...Object.values(loaded)])) { + for (const Table of Object.values(database ?? {})) (Table as any)?._armCleanupScan?.(); + } +} /** * Backfills the stamp of a generation created on a build that stored none, once a peer's stamped definition * proves which generation it is. Only an absent stamp is written; false when one exists or the table is gone. @@ -2158,8 +2192,8 @@ function initStores( const audit = typeof primaryAttribute.audit === 'boolean' ? primaryAttribute.audit : envGet(CONFIG_PARAMS.LOGGING_AUDITLOG); const trackDeletes = primaryAttribute.trackDeletes; - const expiration = primaryAttribute.expiration; - const eviction = primaryAttribute.eviction; + const expiration = loadedTtlSeconds(databaseName, tableName, 'expiration', primaryAttribute.expiration); + const eviction = loadedTtlSeconds(databaseName, tableName, 'eviction', primaryAttribute.eviction); const replicaExpiration = primaryAttribute.replicaExpiration; const replicaRefreshFraction = primaryAttribute.replicaRefreshFraction; const sealed = primaryAttribute.sealed; @@ -2905,9 +2939,10 @@ interface TableDefinition { path?: string; /** Kept when a peer's definition creates the table here, so every copy of a generation shares one stamp. */ createdTime?: number; - expiration?: number; - eviction?: number; - scanInterval?: number; + /** Seconds; 0 or null clears a saved value, undefined (a caller not authoring the schema) keeps it. */ + expiration?: number | null; + eviction?: number | null; + scanInterval?: number | null; /** RocksDB-only replica lifetime in seconds; 0 disables admission. LMDB always proxies. */ replicaExpiration?: number; /** RocksDB-only renewal fraction in (0, 1]; 1 never renews. */ @@ -4360,18 +4395,29 @@ function declareTable(target: TableTarget, tableDefinition: T if (isBranchIdentity(databaseName)) { throw new ClientError(`'${databaseName}' is in use as a branch store identity and cannot be a database name`); } + // Before anything is opened, created or persisted. A peer's values reach only a table this call creates, as + // a catalog load does (see the end of this function). + if (origin !== 'cluster') { + const qualifiedName = `${databaseName}.${tableName}`; + expiration = ttlSecondsOf(qualifiedName, 'expiration', expiration); + eviction = ttlSecondsOf(qualifiedName, 'eviction', eviction); + scanInterval = ttlSecondsOf(qualifiedName, 'scanInterval', scanInterval); + } const rootStore = target.rootStore(databaseName, tableName); const tables = target.tables(databaseName); logger.trace(`Defining ${tableName} in ${databaseName}`); let Table = tables?.[tableName]; + // what a declaration naming only one of expiration and eviction keeps for the other + const savedTtl = + Table && origin !== 'cluster' && (expiration === undefined) !== (eviction === undefined) + ? catalogExpiration(Table) + : undefined; if (rootStore.status === 'closed') { throw new Error(`Can not use a closed data store for ${tableName}`); } - // A re-declaration persists these to the catalog, where a negative expiration fails every later load, and - // nothing it changes before then (index builds included) is rolled back. + // A re-declaration persists these to the catalog, and nothing it changes before then (index builds included) + // is rolled back. if (Table && origin !== 'cluster') { - if (+expiration < 0 || +eviction < 0) - throw new ClientError(`Expiration and eviction of '${databaseName}.${tableName}' can not be negative`); if (replicaExpiration != null) replicaExpirationMsOf(replicaExpiration); if (replicaRefreshFraction != null) replicaRefreshFractionOf(replicaRefreshFraction); } @@ -5786,10 +5832,15 @@ function declareTable(target: TableTarget, tableDefinition: T // A peer's expiration reaches a table this call created through makeTable, as a catalog load. It never // configures an existing table, and applying it here would mark the TTL as this thread's own declaration. if (origin === 'cluster') expiration = eviction = scanInterval = undefined; + if (savedTtl) { + expiration ??= savedTtl.expiration ?? 0; + eviction ??= savedTtl.eviction ?? 0; + } if ( expiration || eviction || scanInterval || + ((expiration === 0 || eviction === 0) && (Table.expirationMS || Table.evictionMS)) || replicaExpiration !== undefined || replicaRefreshFraction !== undefined || attributes.some((attribute) => attribute.expiresAt) @@ -5833,8 +5884,8 @@ function declareTable(target: TableTarget, tableDefinition: T (typeof audit === 'boolean' && audit !== attributeDescriptor.audit) || (sealed !== undefined && sealed !== attributeDescriptor.sealed) || (replicate !== undefined && replicate !== attributeDescriptor.replicate) || - (expiration && +expiration !== +attributeDescriptor.expiration) || - (eviction && +eviction !== +attributeDescriptor.eviction) || + ttlChanged(expiration, attributeDescriptor.expiration) || + ttlChanged(eviction, attributeDescriptor.eviction) || (replicaExpiration != null && replicaExpiration !== attributeDescriptor.replicaExpiration) || (replicaRefreshFraction != null && replicaRefreshFraction !== attributeDescriptor.replicaRefreshFraction) || attribute.type !== attributeDescriptor.type) @@ -5847,8 +5898,10 @@ function declareTable(target: TableTarget, tableDefinition: T if (audit) Table.enableAuditing(); updatedPrimaryAttribute.audit = audit; } - if (expiration) updatedPrimaryAttribute.expiration = +expiration; - if (eviction) updatedPrimaryAttribute.eviction = +eviction; + if (expiration) updatedPrimaryAttribute.expiration = expiration; + else if (expiration === 0) delete updatedPrimaryAttribute.expiration; + if (eviction) updatedPrimaryAttribute.eviction = eviction; + else if (eviction === 0) delete updatedPrimaryAttribute.eviction; if (replicaExpiration != null) { replicaExpirationMsOf(replicaExpiration); updatedPrimaryAttribute.replicaExpiration = replicaExpiration; diff --git a/resources/defineTable.ts b/resources/defineTable.ts index 2b882fa40c..4ac70ff9de 100644 --- a/resources/defineTable.ts +++ b/resources/defineTable.ts @@ -392,7 +392,8 @@ function compileTypeDef(name: string, shape: Shape, options: DefineTableOptions) const { database, ...tableOptions } = options; // `schemaDefined: true` matches @table so the existing-Table re-assert in `table()` fires on - // every reload (DDL parity: added/removed attributes and index changes are applied). + // every reload (DDL parity: added/removed attributes and index changes are applied), and so does + // clearing an expiration or eviction the options no longer declare. return { table: name, type: name, @@ -401,6 +402,8 @@ function compileTypeDef(name: string, shape: Shape, options: DefineTableOptions) properties, schemaDefined: true, ...tableOptions, + expiration: tableOptions.expiration ?? null, + eviction: tableOptions.eviction ?? null, }; } diff --git a/resources/graphql.ts b/resources/graphql.ts index c077d48ff2..5313238842 100644 --- a/resources/graphql.ts +++ b/resources/graphql.ts @@ -288,6 +288,9 @@ async function processGraphQLSchema( // Boolean directive args arrive as actual booleans; tolerate string forms too. if (typeDef.randomAccessFields !== undefined) typeDef.randomAccessFields = typeDef.randomAccessFields === true || typeDef.randomAccessFields === 'true'; + // likewise for the TTL: null clears an expiration or eviction the directive no longer declares + typeDef.expiration ??= null; + typeDef.eviction ??= null; // schema is authoritative for cacheControl: null (vs undefined) clears a removed directive on reload; // coerce to a header-safe string so a non-string arg or CRLF can't reach the HTTP response typeDef.cacheControl = diff --git a/server/threads/socketRouter.ts b/server/threads/socketRouter.ts index 989b7e801e..0cd0918024 100644 --- a/server/threads/socketRouter.ts +++ b/server/threads/socketRouter.ts @@ -21,6 +21,7 @@ import { startTransactionLogCooling } from '../transactionLogCooling.ts'; import { applyConfiguredV8Flags } from './v8Flags.ts'; import { startLongLivedTransactionReporting } from '../../resources/longLivedTransactions.ts'; import { initializeFullTextQueryPolicy } from '../../resources/indexes/fullTextNativeBinding.ts'; +import { armTableCleanupScans } from '../../resources/databases.ts'; import { effectiveHttpThreadCount, isWorkerPoolActive, @@ -108,6 +109,8 @@ export async function startHTTPThreads(threadCount = 2, dynamicThreads?: boolean if (isProcessShuttingDown()) return []; await threadServer.startServers(); if (isProcessShuttingDown()) return []; + // After the components' declarations, so a stale saved TTL never arms a scan that a schema replaces + armTableCleanupScans(); // startServers() schedules listener startup after loading components; await its cached // batch so a bind failure reaches bin/run.ts and exits non-zero in single-thread mode too. await threadServer.listenOnPorts(); diff --git a/unitTests/resources/createTableCatalogOrder.test.js b/unitTests/resources/createTableCatalogOrder.test.js index f686292927..594fc32aaf 100644 --- a/unitTests/resources/createTableCatalogOrder.test.js +++ b/unitTests/resources/createTableCatalogOrder.test.js @@ -191,7 +191,11 @@ describe('create table catalog write order', () => { 'a create that fails before makeTable must not register the class' ); assert.strictEqual(releasedReclamationHandlers.length, 0, 'nothing was registered before makeTable'); - assert.throws(() => table({ ...definition(), expiration: -1 }), /Expiration can not be negative/); + // a peer's definition is not validated at the declaration boundary, so its bad value fails in makeTable + assert.throws( + () => table({ ...definition(), origin: 'cluster', expiration: -1 }), + /must be a non-negative number of seconds/ + ); assert.strictEqual( databases.test[tableName], undefined, diff --git a/unitTests/resources/tableExpirationPersistence.test.js b/unitTests/resources/tableExpirationPersistence.test.js index 2c6916d541..2954e5b211 100644 --- a/unitTests/resources/tableExpirationPersistence.test.js +++ b/unitTests/resources/tableExpirationPersistence.test.js @@ -10,6 +10,8 @@ const { resetDatabases, table, } = require('#src/resources/databases'); +const { defineTable, types } = require('#src/resources/defineTable'); +const { loadGQLSchema } = require('#src/resources/graphql'); const DATABASE = 'test'; @@ -245,4 +247,169 @@ describe('table-level expiration persistence (harper-pro#1179)', () => { assert.strictEqual(Local.evictionMS, 0); assert.deepStrictEqual(catalogExpiration(Local), { expiration: 600 }); }); + it('clears a saved expiration and eviction that a re-declaration sets to 0, null or "0" (#3934)', async function () { + for (const [suffix, cleared] of [ + ['Zero', { expiration: 0, eviction: 0 }], + ['Null', { expiration: null, eviction: null }], + ['String', { expiration: '0', eviction: '0' }], + ]) { + const tableName = `ExpiryCleared${suffix}`; + const Declared = keyed(tableName, { expiration: 600, eviction: 300 }); + keyed(tableName, cleared); + await commitCatalog(Declared); + const { value } = primaryRow(Declared); + assert(!('expiration' in value) && !('eviction' in value), `${suffix}: ${JSON.stringify(value)}`); + assert.deepStrictEqual(catalogExpiration(Declared), {}, suffix); + assert(!Declared.expirationMS, `${suffix}: this thread still applies ${Declared.expirationMS}ms`); + assert.strictEqual(Declared.evictionMS, 0, suffix); + + delete databases[DATABASE][tableName]; + resetDatabases(); + const Restarted = getDatabases()[DATABASE][tableName]; + assert(!Restarted.expirationMS, `${suffix}: a restart loads ${Restarted.expirationMS}ms`); + assert.strictEqual(Restarted.evictionMS, 0, suffix); + } + }); + + it('clears only the setting a re-declaration sets to 0', async function () { + const tableName = 'ExpiryClearedEvictionOnly'; + const Declared = keyed(tableName, { expiration: 600, eviction: 300 }); + keyed(tableName, { expiration: 600, eviction: 0 }); + await commitCatalog(Declared); + assert.deepStrictEqual(catalogExpiration(Declared), { expiration: 600 }); + assert.strictEqual(Declared.evictionMS, 0); + }); + + it('drops a TTL this thread loaded from the catalog when its re-declaration clears it', async function () { + const tableName = 'ExpiryLoadedThenCleared'; + const Loaded = keyed(tableName); + await writeCatalogSettings(Loaded, { expiration: 120, eviction: 60 }); + resetDatabases(); + assert.strictEqual(Loaded.expirationMS, 120000); + + keyed(tableName, { expiration: 0 }); + await commitCatalog(Loaded); + assert(!Loaded.expirationMS, `this thread still applies ${Loaded.expirationMS}ms`); + assert.deepStrictEqual(catalogExpiration(Loaded), { eviction: 60 }, 'an omitted eviction stays saved'); + assert.strictEqual(Loaded.evictionMS, 60000, 'and stays applied'); + }); + + it('keeps the saved value of the setting a re-declaration omits, on disk and in memory', async function () { + const tableName = 'ExpiryPartialUpdate'; + const Declared = keyed(tableName, { expiration: 60, eviction: 3600 }); + keyed(tableName, { expiration: 120 }); + await commitCatalog(Declared); + assert.deepStrictEqual(catalogExpiration(Declared), { expiration: 120, eviction: 3600 }); + assert.strictEqual(Declared.expirationMS, 120000); + assert.strictEqual(Declared.evictionMS, 3600000); + }); + + it('does not clear a TTL application code configured when the schema declares none', async function () { + const tableName = 'ExpiryApplicationKept'; + const Configured = keyed(tableName); + Configured.setTTLExpiration({ expiration: 30, eviction: 10 }); + keyed(tableName, { expiration: null, eviction: null }); + assert.strictEqual(Configured.expirationMS, 30000); + assert.strictEqual(Configured.evictionMS, 10000); + }); + + it('loads a saved value no declaration could save now as absent', async function () { + const tableName = 'ExpiryLegacyInvalid'; + const Legacy = keyed(tableName); + await writeCatalogSettings(Legacy, { expiration: 'soon', eviction: -1 }); + delete databases[DATABASE][tableName]; + resetDatabases(); + const Restarted = getDatabases()[DATABASE][tableName]; + assert(Restarted, 'the table must still load'); + assert(!Restarted.expirationMS); + assert.strictEqual(Restarted.evictionMS, 0); + keyed(tableName, { expiration: 0, eviction: 0 }); + await commitCatalog(Restarted); + const { value } = primaryRow(Restarted); + assert(!('expiration' in value) && !('eviction' in value), 'the next declaration rewrites the row'); + }); + + it('brings a loaded class up to date when another thread clears the catalog (#3934)', async function () { + const tableName = 'ExpiryAdoptedClear'; + const Loaded = keyed(tableName); + await writeCatalogSettings(Loaded, { expiration: 120, eviction: 60 }); + resetDatabases(); + assert.strictEqual(Loaded.expirationMS, 120000); + + const { key, value } = primaryRow(Loaded); + const cleared = { ...value }; + delete cleared.expiration; + delete cleared.eviction; + Loaded.dbisDB.putSync(key, cleared); + await commitCatalog(Loaded); + resetDatabases(); + assert(!Loaded.expirationMS, `the reload kept ${Loaded.expirationMS}ms`); + assert.strictEqual(Loaded.evictionMS, 0); + }); + + it('keeps a saved TTL through callers that do not author it', async function () { + const tableName = 'ExpiryAddAttribute'; + const Declared = keyed(tableName, { expiration: 600, eviction: 300 }); + await Declared.addAttributes([{ name: 'added', type: 'String' }]); + await commitCatalog(Declared); + assert.deepStrictEqual(catalogExpiration(Declared), { expiration: 600, eviction: 300 }); + assert.strictEqual(Declared.expirationMS, 600000); + }); + + it('clears a TTL that a @table directive no longer declares (#3934)', async function () { + const tableName = 'ExpiryGraphqlRemoved'; + await loadGQLSchema(`type ${tableName} @table(database: "${DATABASE}", expiration: 600, eviction: 300) { + id: ID @primaryKey + }`); + const Declared = databases[DATABASE][tableName]; + await commitCatalog(Declared); + assert.deepStrictEqual(catalogExpiration(Declared), { expiration: 600, eviction: 300 }); + + await loadGQLSchema(`type ${tableName} @table(database: "${DATABASE}") { + id: ID @primaryKey + }`); + await commitCatalog(Declared); + assert.deepStrictEqual(catalogExpiration(Declared), {}); + assert(!Declared.expirationMS, `this thread still applies ${Declared.expirationMS}ms`); + assert.strictEqual(Declared.evictionMS, 0); + }); + + it('clears a TTL that defineTable options no longer declare (#3934)', async function () { + const tableName = 'ExpiryDefineTableRemoved'; + const shape = { id: types.id.primaryKey }; + const Declared = defineTable(tableName, shape, { database: DATABASE, expiration: 600 }); + await commitCatalog(Declared); + assert.deepStrictEqual(catalogExpiration(Declared), { expiration: 600 }); + + defineTable(tableName, shape, { database: DATABASE }); + await commitCatalog(Declared); + assert.deepStrictEqual(catalogExpiration(Declared), {}); + assert(!Declared.expirationMS, `this thread still applies ${Declared.expirationMS}ms`); + }); + + it('rejects an invalid TTL before a create, naming the table and field (#1286, #3935)', async function () { + for (const [field, value] of [ + ['expiration', -1], + ['eviction', -1], + ['scanInterval', -1], + ['expiration', 'soon'], + ['eviction', Infinity], + ['expiration', Number.NaN], + ['expiration', 1e308], + ['expiration', ''], + ['expiration', true], + ['eviction', [5]], + ]) { + const tableName = `ExpiryInvalidCreate_${field}_${String(value).replace(/\W/g, '')}`; + assert.throws( + () => keyed(tableName, { [field]: value }), + (error) => + error.statusCode === 400 && + error.message.includes(field) && + error.message.includes(`${DATABASE}.${tableName}`), + `${field}: ${value}` + ); + assert.strictEqual(databases[DATABASE][tableName], undefined, `${field}: ${value} created the table`); + } + }); }); diff --git a/unitTests/resources/ttlCleanupInterval.test.js b/unitTests/resources/ttlCleanupInterval.test.js index f224328d86..1e83b1bff0 100644 --- a/unitTests/resources/ttlCleanupInterval.test.js +++ b/unitTests/resources/ttlCleanupInterval.test.js @@ -1,15 +1,50 @@ require('../testUtils'); const assert = require('assert'); const { setupTestDBPath } = require('../testUtils'); -const { table } = require('#src/resources/databases'); +const { table, databases, getDatabases, resetDatabases, armTableCleanupScans } = require('#src/resources/databases'); const { makeTable } = require('#src/resources/Table'); const { setMainIsWorker } = require('#src/server/threads/manageThreads'); const { waitFor } = require('../waitFor'); const { setTimeout: sleep } = require('node:timers/promises'); +// the delays of the timers armed while `run` executes +function timersArmedBy(run) { + const originalSetTimeout = global.setTimeout; + const delays = []; + global.setTimeout = (callback, delay, ...args) => { + delays.push(delay); + return originalSetTimeout(callback, delay, ...args); + }; + try { + run(); + } finally { + global.setTimeout = originalSetTimeout; + } + return delays; +} + // The cleanup scan is what removes an expired record: reads return it until then. Its interval has to come // from the table's TTL whichever way the TTL was set. A worker that learns of a table from the catalog (a // table created at runtime on another thread, or any table after a restart) arms it with the number form. +// the timers armed while `run` and what it starts within `settleMs` execute +async function timersArmedWithin(run, settleMs) { + const originalSetTimeout = global.setTimeout; + const delays = []; + global.setTimeout = (callback, delay, ...args) => { + delays.push(delay); + return originalSetTimeout(callback, delay, ...args); + }; + try { + run(); + await sleep(settleMs); + } finally { + global.setTimeout = originalSetTimeout; + } + return delays; +} +// a rescheduled scan runs once at once, then at its interval; the daily default is the only one over an hour here +const isDailyCadence = (delay) => delay > 3_600_000 && delay <= 86_400_000; + describe('TTL cleanup scan interval', () => { before(function () { setupTestDBPath(); @@ -154,4 +189,149 @@ describe('TTL cleanup scan interval', () => { Hydrated.cleanup(); } }); + it('an eviction-only TTL adopted from the catalog arms the scan (#3935)', async () => { + const Declared = table({ + table: 'TtlAdoptedEvictionOnly', + database: databaseName, + attributes: [{ name: 'id', isPrimaryKey: true }], + }); + const Hydrated = hydrate(Declared, 'TtlAdoptedEvictionOnly', {}); + try { + const delays = timersArmedBy(() => Hydrated._adoptCatalogExpiration(0, 0.05)); + assert.strictEqual(Hydrated.expirationMS, 0); + assert.strictEqual(Hydrated.evictionMS, 50); + assert.strictEqual(delays.length, 1, 'adopting an eviction-only TTL must arm the scan'); + await Hydrated.put(1, { id: 1 }, { expiresAt: Date.now() }); + await Hydrated.primaryStore.committed; + await waitFor(() => !stored(Hydrated, 1), { timeout: 5000, message: 'the evictable record was never evicted' }); + } finally { + Hydrated.cleanup(); + } + }); + + it('adopting a cleared TTL leaves the scan at the daily default (#3934)', async () => { + const Declared = table({ + table: 'TtlAdoptedClear', + database: databaseName, + attributes: [{ name: 'id', isPrimaryKey: true }], + }); + const Hydrated = hydrate(Declared, 'TtlAdoptedClear', { expirationMS: 60_000 }); + try { + const delays = await timersArmedWithin(() => Hydrated._adoptCatalogExpiration(undefined, undefined), 200); + assert.strictEqual(Hydrated.expirationMS, 0); + assert.strictEqual(Hydrated.evictionMS, 0); + // the armed scan moves to the daily cadence a table with no TTL keeps for per-record expiry + assert(delays.some(isDailyCadence), `no daily scan armed: ${delays}`); + } finally { + Hydrated.cleanup(); + } + }); + + it('clearing a TTL with zeros keeps a scan for per-record expiry instead of disabling it', async () => { + const Table = table({ + table: 'TtlZeroed', + database: databaseName, + attributes: [{ name: 'id', isPrimaryKey: true }], + }); + try { + Table.setTTLExpiration({ expiration: 60 }); + const delays = await timersArmedWithin(() => Table.setTTLExpiration({ expiration: 0, eviction: 0 }), 200); + assert(delays.some(isDailyCadence), `the scan was disabled instead of moved to the daily default: ${delays}`); + } finally { + Table.cleanup(); + } + }); + + it('rejects a negative or non-finite value without applying any of it (#3935)', () => { + const Table = table({ + table: 'TtlRejected', + database: databaseName, + attributes: [{ name: 'id', isPrimaryKey: true }], + }); + try { + Table.setTTLExpiration({ expiration: 10, eviction: 5 }); + for (const opts of [ + { expiration: 1, eviction: -1 }, + { expiration: -1, eviction: 1 }, + { expiration: 1, scanInterval: -1 }, + { expiration: Infinity }, + { expiration: Number.NaN }, + -1, + ]) { + assert.throws( + () => Table.setTTLExpiration(opts), + /must be a non-negative number of seconds/, + JSON.stringify(opts) + ); + assert.strictEqual(Table.expirationMS, 10_000, `${JSON.stringify(opts)} changed the expiration`); + assert.strictEqual(Table.evictionMS, 5_000, `${JSON.stringify(opts)} changed the eviction`); + } + } finally { + Table.cleanup(); + } + }); + + it('a table loaded before the main thread became a worker arms its scan once it is one (#2831)', async () => { + const tableName = 'TtlLoadedBeforePromotion'; + const Declared = table({ + table: tableName, + database: databaseName, + expiration: 0.05, + attributes: [{ name: 'id', isPrimaryKey: true }], + }); + // a restart: the row outlives this class, whose scan stops with it + await Declared.put(1, { id: 1 }, { expiresAt: Date.now() + 100 }); + await Declared.primaryStore.committed; + Declared.cleanup(); + delete databases[databaseName][tableName]; + let Loaded; + try { + // threads: 0 loads every table on the main thread before setMainIsWorker(true) + setMainIsWorker(false); + resetDatabases(); + Loaded = getDatabases()[databaseName][tableName]; + } finally { + setMainIsWorker(true); + } + try { + assert.strictEqual(Loaded.expirationMS, 50); + await sleep(400); + assert(stored(Loaded, 1), 'a scan ran although no thread owned it at load'); + armTableCleanupScans(); + await waitFor(() => !stored(Loaded, 1), { timeout: 5000, message: 'the expired record was never evicted' }); + } finally { + Loaded.cleanup(); + } + }); + + it('arms no scan for a table with no TTL', () => { + const Table = table({ + table: 'TtlNoneToArm', + database: databaseName, + attributes: [{ name: 'id', isPrimaryKey: true }], + }); + try { + assert.deepStrictEqual( + timersArmedBy(() => Table._armCleanupScan()), + [] + ); + } finally { + Table.cleanup(); + } + }); + + it('arms system tables too, which databases does not enumerate', () => { + const systemTables = Object.values(databases.system); + assert(systemTables.length > 0); + assert(!Object.keys(databases).includes('system'), 'the system database is enumerable now; simplify this test'); + const armed = new Set(); + const originals = systemTables.map((Table) => [Table, Table._armCleanupScan]); + for (const Table of systemTables) Table._armCleanupScan = () => armed.add(Table); + try { + armTableCleanupScans(); + } finally { + for (const [Table, original] of originals) Table._armCleanupScan = original; + } + assert.strictEqual(armed.size, systemTables.length); + }); }); From d7f3caf69db50256d3d3a39be6eec9da1ff898f3 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Tue, 10 Nov 2026 21:44:03 -0700 Subject: [PATCH 02/11] Take an omitted TTL field from the row the declaration writes; render bad saved values safely - A table() declaration naming only one of expiration and eviction now takes the other from the catalog row persistTableSettings reads (re-read under the schema lock when it writes), not from a read before the lock, so a concurrent declaration's value is not written back stale. - The warning for an invalid saved TTL renders any value without throwing and is marked sent only after it is logged; saved values other than numbers and numeric strings load as absent. - Docs say which fields a schema clears and which threads adopt the saved value. - Trimmed comments that narrated the code. Co-Authored-By: Claude Opus 5.5 Dispatch-Task: proposal-kriszyp-bundle-harper-table-expiration-config-5df9b5 --- dataLayer/schemaDescribe.ts | 1 - docs/reference/database/schema.md | 4 +- .../removing-a-table-expiration-clears-it.md | 2 +- .../database/table-expiration-config.test.ts | 2 - resources/Table.ts | 18 +++++---- resources/databases.ts | 24 +++++++----- resources/defineTable.ts | 3 +- .../tableExpirationPersistence.test.js | 29 +++++++++++++- .../resources/ttlCleanupInterval.test.js | 38 +++++-------------- 9 files changed, 65 insertions(+), 56 deletions(-) diff --git a/dataLayer/schemaDescribe.ts b/dataLayer/schemaDescribe.ts index 68aa06dbd5..64bf24280e 100644 --- a/dataLayer/schemaDescribe.ts +++ b/dataLayer/schemaDescribe.ts @@ -222,7 +222,6 @@ async function descTable(describeTableObject: any, attrPerms?: any) { db_size, }; if (tableObj.replicate !== undefined) tableResult.replicate = tableObj.replicate; - // an eviction-only table reports 0s; a cleared one, like one that never had a TTL, reports none if (tableObj.expirationMS || tableObj.evictionMS) tableResult.expiration = tableObj.expirationMS / 1000 + 's'; if (tableObj.sealed !== undefined) tableResult.sealed = tableObj.sealed; if (tableObj.cacheControl != null) tableResult.cacheControl = tableObj.cacheControl; diff --git a/docs/reference/database/schema.md b/docs/reference/database/schema.md index 3e93ad9bd8..1d20254b14 100644 --- a/docs/reference/database/schema.md +++ b/docs/reference/database/schema.md @@ -109,9 +109,9 @@ type WeatherCache @table(expiration: 300, eviction: 3300, scanInterval: 600) { **Changing or removing them** -`expiration` and `eviction` are saved with the table, so every thread and every restart applies what the schema last declared. Removing `expiration` or `eviction` from `@table` (or from the `defineTable` options), or setting it to `0`, clears the saved value: records written afterwards no longer expire, and records already written keep the expiry they were given. Before v5.4.0 a removed value stayed in effect until the table was dropped. A TTL that application code sets at runtime (for example through `sourcedFrom` options) is not cleared by a schema that declares none. +`expiration` and `eviction` are saved with the table, and a restart applies what the schema last declared. `@table` (and `defineTable`) sets each of the two that it names and clears each one it omits, including a value `create_table` saved; setting one to `0` clears it too. Without `expiration`, records written afterwards no longer expire; without `eviction`, they are removed as soon as they expire. Records already written keep the expiry they were given. Before v5.4.0 an omitted value stayed in effect until the table was dropped. A TTL that application code sets at runtime (for example through `sourcedFrom` options) is not cleared by a schema that declares none. -Each value must be a number of seconds, zero or more. A negative or non-numeric value fails the declaration (or the `create_table` operation) with an error naming the table and the argument, and the table is left unchanged. +Each value must be a number of seconds, zero or more. A negative or non-numeric value fails the declaration with an error naming the table and the argument, and the table is left unchanged. #### How `scanInterval` Determines the Eviction Cycle diff --git a/docs/release-notes/v5-lincoln/5.4/removing-a-table-expiration-clears-it.md b/docs/release-notes/v5-lincoln/5.4/removing-a-table-expiration-clears-it.md index 6a167980ba..f4a664eea6 100644 --- a/docs/release-notes/v5-lincoln/5.4/removing-a-table-expiration-clears-it.md +++ b/docs/release-notes/v5-lincoln/5.4/removing-a-table-expiration-clears-it.md @@ -4,4 +4,4 @@ section: 'Storage' ### Removing a Table's Expiration Clears It -Removing `expiration` or `eviction` from `@table` (or from `defineTable` options), or setting it to `0`, now clears the saved value on every thread and after a restart. Before, the old value stayed in effect, so a table kept expiring records after its schema stopped declaring a TTL. If you relied on that, declare the value again. Records already written keep the expiry they were given. A negative or non-numeric `expiration`, `eviction` or `scanInterval` now fails the declaration or `create_table` with an error naming the table and argument, instead of failing the table later; `create_table` also accepts `eviction`. Expired records are now removed on schedule after a restart with `threads.count: 0`, and on a table that has only `eviction`, without waiting for a write. See [`@table`](/reference/v5/database/schema#table). +`@table` (and `defineTable`) now clears an `expiration` or `eviction` it no longer names, or sets to `0`, after a restart and on threads that load the table without declaring it. Before, the old value stayed in effect, so a table kept expiring records after its schema stopped declaring a TTL. If you relied on that, declare the value again. Records already written keep the expiry they were given. A negative or non-numeric `expiration`, `eviction` or `scanInterval` now fails the declaration with an error naming the table and argument, instead of failing the table later; `create_table` accepts `eviction` and rejects a negative or non-numeric `expiration` or `eviction` with a 400. Expired records are now removed on schedule after a restart with `threads.count: 0`, and on a table that has only `eviction`, without waiting for a write. See [`@table`](/reference/v5/database/schema#table). diff --git a/integrationTests/database/table-expiration-config.test.ts b/integrationTests/database/table-expiration-config.test.ts index b79948dcdf..702d825c60 100644 --- a/integrationTests/database/table-expiration-config.test.ts +++ b/integrationTests/database/table-expiration-config.test.ts @@ -71,7 +71,6 @@ async function storedRows(ctx: ContextWithHarper, table: string): Promise, dueAt: number) { await killHarper(ctx); await dropDeclaredTtl(ctx); diff --git a/resources/Table.ts b/resources/Table.ts index f02a6d0476..261b72242d 100644 --- a/resources/Table.ts +++ b/resources/Table.ts @@ -826,19 +826,21 @@ export function ttlSecondsOf(table: string, field: string, value: unknown): numb const seconds = typeof value === 'number' ? value : typeof value === 'string' && value.trim() !== '' ? Number(value) : NaN; if (seconds >= 0 && seconds <= MAX_TTL_SECONDS) return seconds; - const shown = - typeof value === 'number' - ? String(value) - : typeof value === 'string' - ? JSON.stringify(value.slice(0, 40)) - : typeof value; - throw new ClientError(`${field} of '${table}' must be a non-negative number of seconds, got ${shown}`); + throw new ClientError( + `${field} of '${table}' must be a non-negative number of seconds, got ${describeTtlValue(value)}` + ); } /** A TTL field as the catalog holds it: absent unless a valid positive number of seconds. */ export function savedTtlSeconds(value: unknown): number | undefined { - const seconds = Number(value); + const seconds = typeof value === 'number' || typeof value === 'string' ? Number(value) : NaN; return seconds > 0 && seconds <= MAX_TTL_SECONDS ? seconds : undefined; } +/** A rejected TTL value for a message: bounded, and never a value that cannot be rendered. */ +export function describeTtlValue(value: unknown): string { + if (typeof value === 'number' || typeof value === 'bigint' || typeof value === 'boolean') return String(value); + if (typeof value === 'string') return JSON.stringify(value.slice(0, 40)); + return value === null ? 'null' : typeof value; +} const TEST_WRITE_KEY_BUFFER = Buffer.allocUnsafeSlow(8192); const MAX_KEY_BYTES = 1978; const EVENT_HIGH_WATER_MARK = 100; diff --git a/resources/databases.ts b/resources/databases.ts index efde371cf7..92ad5e0310 100644 --- a/resources/databases.ts +++ b/resources/databases.ts @@ -44,6 +44,7 @@ import { makeTable, replicaExpirationMsOf, replicaRefreshFractionOf, + describeTtlValue, savedTtlSeconds, ttlSecondsOf, ignoreAlreadyDropped, @@ -1139,6 +1140,9 @@ export function catalogCreatedTime(table: { dbisDB: any; tableName: string; prim const createdTime = primaryCatalogRowFor(table)?.value?.createdTime; return Number.isFinite(createdTime) ? createdTime : undefined; } +function savedTtlOf(row: any): { expiration?: number; eviction?: number } { + return { expiration: savedTtlSeconds(row.expiration), eviction: savedTtlSeconds(row.eviction) }; +} /** Whether a declared TTL field (undefined = not declared, 0 = cleared) differs from the catalog's. */ function ttlChanged(declared: number | undefined, saved: unknown): boolean { if (declared === undefined) return false; @@ -1153,10 +1157,10 @@ function loadedTtlSeconds(databaseName: string, tableName: string, field: string const seconds = savedTtlSeconds(saved); const key = `${databaseName}.${tableName}.${field}`; if (seconds === undefined && saved !== undefined && saved !== 0 && !warnedInvalidSavedTtl.has(key)) { - warnedInvalidSavedTtl.add(key); logger.warn( - `Ignoring the saved ${field} ${JSON.stringify(saved)} of ${databaseName}.${tableName}: not a valid TTL` + `Ignoring the saved ${field} ${describeTtlValue(saved)} of ${databaseName}.${tableName}: not a valid TTL` ); + warnedInvalidSavedTtl.add(key); } return seconds; } @@ -4407,11 +4411,9 @@ function declareTable(target: TableTarget, tableDefinition: T const tables = target.tables(databaseName); logger.trace(`Defining ${tableName} in ${databaseName}`); let Table = tables?.[tableName]; - // what a declaration naming only one of expiration and eviction keeps for the other - const savedTtl = - Table && origin !== 'cluster' && (expiration === undefined) !== (eviction === undefined) - ? catalogExpiration(Table) - : undefined; + const namesOneTtlField = origin !== 'cluster' && (expiration === undefined) !== (eviction === undefined); + // the saved value of the field such a declaration omits, from the row persistTableSettings writes + let inheritedTtl: { expiration?: number; eviction?: number } | undefined; if (rootStore.status === 'closed') { throw new Error(`Can not use a closed data store for ${tableName}`); } @@ -5832,9 +5834,9 @@ function declareTable(target: TableTarget, tableDefinition: T // A peer's expiration reaches a table this call created through makeTable, as a catalog load. It never // configures an existing table, and applying it here would mark the TTL as this thread's own declaration. if (origin === 'cluster') expiration = eviction = scanInterval = undefined; - if (savedTtl) { - expiration ??= savedTtl.expiration ?? 0; - eviction ??= savedTtl.eviction ?? 0; + if (inheritedTtl) { + expiration ??= inheritedTtl.expiration ?? 0; + eviction ??= inheritedTtl.eviction ?? 0; } if ( expiration || @@ -5870,6 +5872,7 @@ function declareTable(target: TableTarget, tableDefinition: T } function persistTableSettings(attribute: any, dbiKey: string) { const attributeDescriptor = attributesDbi.getSync(dbiKey) || {}; + if (namesOneTtlField) inheritedTtl = savedTtlOf(attributeDescriptor); // Persist schemaDefined when the explicit live value disagrees with disk. Without this, // a stale `false` (from a v4-era write or replicated event) survives every reload: the // in-memory re-assert in the existing-Table branch only fixes the worker that ran @table, @@ -5893,6 +5896,7 @@ function declareTable(target: TableTarget, tableDefinition: T exclusiveLock(); const currentPrimaryAttribute = attributesDbi.getSync(dbiKey); if (!currentPrimaryAttribute || tableIsDropping(currentPrimaryAttribute, dbiKey)) return; + if (namesOneTtlField) inheritedTtl = savedTtlOf(currentPrimaryAttribute); const updatedPrimaryAttribute = { ...currentPrimaryAttribute }; if (typeof audit === 'boolean') { if (audit) Table.enableAuditing(); diff --git a/resources/defineTable.ts b/resources/defineTable.ts index 4ac70ff9de..726b577000 100644 --- a/resources/defineTable.ts +++ b/resources/defineTable.ts @@ -392,8 +392,7 @@ function compileTypeDef(name: string, shape: Shape, options: DefineTableOptions) const { database, ...tableOptions } = options; // `schemaDefined: true` matches @table so the existing-Table re-assert in `table()` fires on - // every reload (DDL parity: added/removed attributes and index changes are applied), and so does - // clearing an expiration or eviction the options no longer declare. + // every reload (DDL parity: added/removed attributes and index changes are applied). return { table: name, type: name, diff --git a/unitTests/resources/tableExpirationPersistence.test.js b/unitTests/resources/tableExpirationPersistence.test.js index 2954e5b211..5efcfb1721 100644 --- a/unitTests/resources/tableExpirationPersistence.test.js +++ b/unitTests/resources/tableExpirationPersistence.test.js @@ -304,6 +304,33 @@ describe('table-level expiration persistence (harper-pro#1179)', () => { assert.strictEqual(Declared.evictionMS, 3600000); }); + it('keeps the omitted setting as the row it writes holds it, not as an earlier read saw it', async function () { + const tableName = 'ExpiryInheritedUnderLock'; + const Declared = keyed(tableName, { expiration: 60, eviction: 300 }); + await commitCatalog(Declared); + const { key } = primaryRow(Declared); + const catalog = Declared.dbisDB; + const originalGetSync = catalog.getSync; + let raced = false; + catalog.getSync = function (readKey, ...rest) { + const value = originalGetSync.call(this, readKey, ...rest); + if (readKey === key && !raced) { + raced = true; + // another thread saves a new eviction right after this declaration's first read of the row + catalog.putSync(key, { ...value, eviction: 3600 }); + } + return value; + }; + try { + keyed(tableName, { expiration: 120 }); + } finally { + catalog.getSync = originalGetSync; + } + await commitCatalog(Declared); + assert.deepStrictEqual(catalogExpiration(Declared), { expiration: 120, eviction: 3600 }); + assert.strictEqual(Declared.evictionMS, 3600000); + }); + it('does not clear a TTL application code configured when the schema declares none', async function () { const tableName = 'ExpiryApplicationKept'; const Configured = keyed(tableName); @@ -316,7 +343,7 @@ describe('table-level expiration persistence (harper-pro#1179)', () => { it('loads a saved value no declaration could save now as absent', async function () { const tableName = 'ExpiryLegacyInvalid'; const Legacy = keyed(tableName); - await writeCatalogSettings(Legacy, { expiration: 'soon', eviction: -1 }); + await writeCatalogSettings(Legacy, { expiration: { legacy: 1n }, eviction: -1 }); delete databases[DATABASE][tableName]; resetDatabases(); const Restarted = getDatabases()[DATABASE][tableName]; diff --git a/unitTests/resources/ttlCleanupInterval.test.js b/unitTests/resources/ttlCleanupInterval.test.js index 1e83b1bff0..8b40ea30cd 100644 --- a/unitTests/resources/ttlCleanupInterval.test.js +++ b/unitTests/resources/ttlCleanupInterval.test.js @@ -7,8 +7,7 @@ const { setMainIsWorker } = require('#src/server/threads/manageThreads'); const { waitFor } = require('../waitFor'); const { setTimeout: sleep } = require('node:timers/promises'); -// the delays of the timers armed while `run` executes -function timersArmedBy(run) { +async function timersArmedBy(run, settleMs = 0) { const originalSetTimeout = global.setTimeout; const delays = []; global.setTimeout = (callback, delay, ...args) => { @@ -17,34 +16,18 @@ function timersArmedBy(run) { }; try { run(); + if (settleMs) await sleep(settleMs); } finally { global.setTimeout = originalSetTimeout; } return delays; } +// a rescheduled scan runs once at once, then at its interval; the daily default is the only one over an hour here +const isDailyCadence = (delay) => delay > 3_600_000 && delay <= 86_400_000; // The cleanup scan is what removes an expired record: reads return it until then. Its interval has to come // from the table's TTL whichever way the TTL was set. A worker that learns of a table from the catalog (a // table created at runtime on another thread, or any table after a restart) arms it with the number form. -// the timers armed while `run` and what it starts within `settleMs` execute -async function timersArmedWithin(run, settleMs) { - const originalSetTimeout = global.setTimeout; - const delays = []; - global.setTimeout = (callback, delay, ...args) => { - delays.push(delay); - return originalSetTimeout(callback, delay, ...args); - }; - try { - run(); - await sleep(settleMs); - } finally { - global.setTimeout = originalSetTimeout; - } - return delays; -} -// a rescheduled scan runs once at once, then at its interval; the daily default is the only one over an hour here -const isDailyCadence = (delay) => delay > 3_600_000 && delay <= 86_400_000; - describe('TTL cleanup scan interval', () => { before(function () { setupTestDBPath(); @@ -197,7 +180,7 @@ describe('TTL cleanup scan interval', () => { }); const Hydrated = hydrate(Declared, 'TtlAdoptedEvictionOnly', {}); try { - const delays = timersArmedBy(() => Hydrated._adoptCatalogExpiration(0, 0.05)); + const delays = await timersArmedBy(() => Hydrated._adoptCatalogExpiration(0, 0.05)); assert.strictEqual(Hydrated.expirationMS, 0); assert.strictEqual(Hydrated.evictionMS, 50); assert.strictEqual(delays.length, 1, 'adopting an eviction-only TTL must arm the scan'); @@ -217,7 +200,7 @@ describe('TTL cleanup scan interval', () => { }); const Hydrated = hydrate(Declared, 'TtlAdoptedClear', { expirationMS: 60_000 }); try { - const delays = await timersArmedWithin(() => Hydrated._adoptCatalogExpiration(undefined, undefined), 200); + const delays = await timersArmedBy(() => Hydrated._adoptCatalogExpiration(undefined, undefined), 200); assert.strictEqual(Hydrated.expirationMS, 0); assert.strictEqual(Hydrated.evictionMS, 0); // the armed scan moves to the daily cadence a table with no TTL keeps for per-record expiry @@ -235,7 +218,7 @@ describe('TTL cleanup scan interval', () => { }); try { Table.setTTLExpiration({ expiration: 60 }); - const delays = await timersArmedWithin(() => Table.setTTLExpiration({ expiration: 0, eviction: 0 }), 200); + const delays = await timersArmedBy(() => Table.setTTLExpiration({ expiration: 0, eviction: 0 }), 200); assert(delays.some(isDailyCadence), `the scan was disabled instead of moved to the daily default: ${delays}`); } finally { Table.cleanup(); @@ -304,17 +287,14 @@ describe('TTL cleanup scan interval', () => { } }); - it('arms no scan for a table with no TTL', () => { + it('arms no scan for a table with no TTL', async () => { const Table = table({ table: 'TtlNoneToArm', database: databaseName, attributes: [{ name: 'id', isPrimaryKey: true }], }); try { - assert.deepStrictEqual( - timersArmedBy(() => Table._armCleanupScan()), - [] - ); + assert.deepStrictEqual(await timersArmedBy(() => Table._armCleanupScan()), []); } finally { Table.cleanup(); } From ba4a5709d95e0da7f67755cade5a2ed690897913 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Tue, 10 Nov 2026 21:58:39 -0700 Subject: [PATCH 03/11] Keep a scan for rows a cleared TTL stamped; validate a peer's eviction on create - A setTTLExpiration call that clears a TTL whose scan was scheduled now arms the daily scan even when no timer exists yet, as under threads: 0 when the schema clears a TTL the main thread loaded before it became worker 0: rows written under the old TTL keep their expiry and still need it. - makeTable arms a loaded TTL through the object form, so a peer definition's negative eviction fails the create like a negative expiration, instead of giving the scan a negative interval. Co-Authored-By: Claude Opus 5.5 Dispatch-Task: proposal-kriszyp-bundle-harper-table-expiration-config-5df9b5 --- .../database/table-expiration-config.test.ts | 1 - resources/Table.ts | 9 +++--- resources/graphql.ts | 3 +- .../tableExpirationPersistence.test.js | 9 ++++++ .../resources/ttlCleanupInterval.test.js | 30 +++++++++++++++++-- 5 files changed, 42 insertions(+), 10 deletions(-) diff --git a/integrationTests/database/table-expiration-config.test.ts b/integrationTests/database/table-expiration-config.test.ts index 702d825c60..5b30be7ec2 100644 --- a/integrationTests/database/table-expiration-config.test.ts +++ b/integrationTests/database/table-expiration-config.test.ts @@ -376,7 +376,6 @@ for (const engine of ['rocksdb', 'lmdb']) { await checks.check(() => waitForWorkerTtl(ctx, 'RuntimeEviction', 0, { expirationMS: 0, evictionMS: RUNTIME_EVICTION * 1000 }) ); - // last before the restart, so the rows only become evictable after it await insertRows(ctx, 'RuntimeTtl'); const expiresInMs = RUNTIME_EXPIRATION * 1000; await untilServed(ctx, { op: 'write', table: 'RuntimeEviction', count: ROWS, expiresInMs }); diff --git a/resources/Table.ts b/resources/Table.ts index 261b72242d..088f702d58 100644 --- a/resources/Table.ts +++ b/resources/Table.ts @@ -4003,6 +4003,8 @@ export function makeTable(options): TableResourceClass { recordScanInterval = (scanIntervalSeconds ?? 0) * 1000; } if (declaredHere && !preserveLoadedConfiguration) ttlDeclaredBySchema = true; + // rows written under a TTL this call clears keep their expiry, so they still need a scan + const scanWasScheduled = !preserveLoadedConfiguration && (expirationScanScheduled || cleanupTimer); if (!preserveLoadedConfiguration) { expirationScanScheduled = Boolean(recordScanInterval || expirationMs || evictionMs); cleanupInterval = recordScanInterval || (expirationMs + evictionMs) / 4 || DEFAULT_CLEANUP_INTERVAL; @@ -4015,9 +4017,8 @@ export function makeTable(options): TableResourceClass { if (replicaExpirationMs) armReplicaEviction(); } else if (replicaEvictionArmed) armReplicaEviction(); // Re-evaluate an existing table-level scan after an ownership-only declaration, but do not - // create the default daily cleanup timer for a table that has only an @expiresAt field. A - // cleared TTL moves an armed scan to the daily default rather than leaving its old cadence. - if (expirationScanScheduled || evictionMs || (!preserveLoadedConfiguration && cleanupTimer)) scheduleCleanup(); + // create the default daily cleanup timer for a table that has only an @expiresAt field. + if (expirationScanScheduled || evictionMs || scanWasScheduled) scheduleCleanup(); // @expiresAt has its own interval rather than the cleanup timer above. Arm it whenever a live // declaration introduces the attribute, including after this application already claimed TTL. if (expiresAtProperty && !recordExpirationInterval) runRecordExpirationEviction(); @@ -12432,7 +12433,7 @@ export function makeTable(options): TableResourceClass { if (expirationMs || evictionMs) { ttlFromLoad = true; try { - TableResource.setTTLExpiration(expirationMs / 1000); + TableResource.setTTLExpiration({ expiration: expirationMs / 1000, eviction: evictionMs / 1000 }); } finally { ttlFromLoad = false; } diff --git a/resources/graphql.ts b/resources/graphql.ts index 5313238842..3cdc66d914 100644 --- a/resources/graphql.ts +++ b/resources/graphql.ts @@ -288,10 +288,9 @@ async function processGraphQLSchema( // Boolean directive args arrive as actual booleans; tolerate string forms too. if (typeDef.randomAccessFields !== undefined) typeDef.randomAccessFields = typeDef.randomAccessFields === true || typeDef.randomAccessFields === 'true'; - // likewise for the TTL: null clears an expiration or eviction the directive no longer declares typeDef.expiration ??= null; typeDef.eviction ??= null; - // schema is authoritative for cacheControl: null (vs undefined) clears a removed directive on reload; + // schema is authoritative for the TTL and cacheControl: null (vs undefined) clears a removed directive on reload; // coerce to a header-safe string so a non-string arg or CRLF can't reach the HTTP response typeDef.cacheControl = typeDef.cacheControl == null diff --git a/unitTests/resources/tableExpirationPersistence.test.js b/unitTests/resources/tableExpirationPersistence.test.js index 5efcfb1721..f9eb1ee1c8 100644 --- a/unitTests/resources/tableExpirationPersistence.test.js +++ b/unitTests/resources/tableExpirationPersistence.test.js @@ -224,6 +224,15 @@ describe('table-level expiration persistence (harper-pro#1179)', () => { assert.strictEqual(Created.expirationMS, 3600000); }); + it("rejects a peer definition's negative eviction on a table it creates, as it does a negative expiration", function () { + const tableName = 'ExpiryPeerNegativeEviction'; + assert.throws( + () => keyed(tableName, { origin: 'cluster', schemaDefined: undefined, eviction: -1 }), + /must be a non-negative number of seconds/ + ); + assert.strictEqual(databases[DATABASE][tableName], undefined); + }); + it("leaves an existing table's settings alone when a peer definition differs", async function () { const tableName = 'ExpiryPeerExisting'; const Local = keyed(tableName, { expiration: 600 }); diff --git a/unitTests/resources/ttlCleanupInterval.test.js b/unitTests/resources/ttlCleanupInterval.test.js index 8b40ea30cd..2b09d4bfe0 100644 --- a/unitTests/resources/ttlCleanupInterval.test.js +++ b/unitTests/resources/ttlCleanupInterval.test.js @@ -22,8 +22,8 @@ async function timersArmedBy(run, settleMs = 0) { } return delays; } -// a rescheduled scan runs once at once, then at its interval; the daily default is the only one over an hour here -const isDailyCadence = (delay) => delay > 3_600_000 && delay <= 86_400_000; +// the TTLs these tests clear scan every 15 s or less +const isDailyCadence = (delay) => delay > 60_000 && delay <= 86_400_000; // The cleanup scan is what removes an expired record: reads return it until then. Its interval has to come // from the table's TTL whichever way the TTL was set. A worker that learns of a table from the catalog (a @@ -203,7 +203,6 @@ describe('TTL cleanup scan interval', () => { const delays = await timersArmedBy(() => Hydrated._adoptCatalogExpiration(undefined, undefined), 200); assert.strictEqual(Hydrated.expirationMS, 0); assert.strictEqual(Hydrated.evictionMS, 0); - // the armed scan moves to the daily cadence a table with no TTL keeps for per-record expiry assert(delays.some(isDailyCadence), `no daily scan armed: ${delays}`); } finally { Hydrated.cleanup(); @@ -287,6 +286,31 @@ describe('TTL cleanup scan interval', () => { } }); + it('a TTL cleared once the main thread is a worker still scans the rows it stamped (#2831, #3934)', async () => { + const Declared = table({ + table: 'TtlClearedAfterPromotion', + database: databaseName, + attributes: [{ name: 'id', isPrimaryKey: true }], + }); + let Hydrated; + try { + // threads: 0 loads the table before the main thread owns any scan, then the schema clears its TTL + setMainIsWorker(false); + Hydrated = hydrate(Declared, 'TtlClearedAfterPromotion', { expirationMS: 60_000 }); + } finally { + setMainIsWorker(true); + } + try { + const delays = await timersArmedBy( + () => Hydrated.setTTLExpiration({ expiration: 0, eviction: 0, fromSchema: true }), + 200 + ); + assert(delays.some(isDailyCadence), `no scan armed for the rows the cleared TTL stamped: ${delays}`); + } finally { + Hydrated.cleanup(); + } + }); + it('arms no scan for a table with no TTL', async () => { const Table = table({ table: 'TtlNoneToArm', From 323edc6468b0fdc6881ee469d30cc1ac933f8cc3 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Tue, 10 Nov 2026 23:03:56 -0700 Subject: [PATCH 04/11] Validate any loaded TTL that is not absent or zero, so a peer's non-numeric value fails its create makeTable now arms a loaded TTL through setTTLExpiration whenever expiration or eviction is set to anything but absent or zero, so a peer definition's NaN, negative or non-numeric value fails the create with an error naming that field, instead of leaving a NaN expiration on the class. An eviction-only value is validated as eviction, not as a NaN expiration. Co-Authored-By: Claude Opus 5.5 Dispatch-Task: proposal-kriszyp-bundle-harper-table-expiration-config-5df9b5 --- .../database/table-expiration-config.test.ts | 2 +- resources/Table.ts | 5 +++-- .../tableExpirationPersistence.test.js | 21 ++++++++++++------- 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/integrationTests/database/table-expiration-config.test.ts b/integrationTests/database/table-expiration-config.test.ts index 5b30be7ec2..5343fdd98d 100644 --- a/integrationTests/database/table-expiration-config.test.ts +++ b/integrationTests/database/table-expiration-config.test.ts @@ -170,7 +170,7 @@ async function assertRowsKept(ctx: ContextWithHarper, table: string, why: string } } -// Every check after a restart runs, so one run on a base that lacks a fix reports each check it fails. +// Every check after a restart runs, so a failure reports all of them, not only the first. function softChecks() { const failures: string[] = []; return { diff --git a/resources/Table.ts b/resources/Table.ts index 088f702d58..65b5b0ea51 100644 --- a/resources/Table.ts +++ b/resources/Table.ts @@ -12430,10 +12430,11 @@ export function makeTable(options): TableResourceClass { defaultAllowReads.add(TableResource.prototype.allowRead); try { TableResource.updatedAttributes(); // on creation, update accessors as well - if (expirationMs || evictionMs) { + // anything but absent or zero is validated here: a peer's definition reaches a create unvalidated + if ((expirationMs !== undefined && expirationMs !== 0) || evictionMs !== 0) { ttlFromLoad = true; try { - TableResource.setTTLExpiration({ expiration: expirationMs / 1000, eviction: evictionMs / 1000 }); + TableResource.setTTLExpiration({ expiration: (expirationMs ?? 0) / 1000, eviction: evictionMs / 1000 }); } finally { ttlFromLoad = false; } diff --git a/unitTests/resources/tableExpirationPersistence.test.js b/unitTests/resources/tableExpirationPersistence.test.js index f9eb1ee1c8..73ace9327a 100644 --- a/unitTests/resources/tableExpirationPersistence.test.js +++ b/unitTests/resources/tableExpirationPersistence.test.js @@ -224,13 +224,20 @@ describe('table-level expiration persistence (harper-pro#1179)', () => { assert.strictEqual(Created.expirationMS, 3600000); }); - it("rejects a peer definition's negative eviction on a table it creates, as it does a negative expiration", function () { - const tableName = 'ExpiryPeerNegativeEviction'; - assert.throws( - () => keyed(tableName, { origin: 'cluster', schemaDefined: undefined, eviction: -1 }), - /must be a non-negative number of seconds/ - ); - assert.strictEqual(databases[DATABASE][tableName], undefined); + it("rejects a peer definition's invalid TTL on a table it creates, naming the field", function () { + for (const [suffix, settings, field] of [ + ['Eviction', { expiration: 60, eviction: -1 }, 'eviction'], + ['EvictionOnly', { eviction: -1 }, 'eviction'], + ['NonNumeric', { expiration: 'soon' }, 'expiration'], + ]) { + const tableName = `ExpiryPeerInvalid${suffix}`; + assert.throws( + () => keyed(tableName, { origin: 'cluster', schemaDefined: undefined, ...settings }), + new RegExp(`^Error: ${field} of '${DATABASE}.${tableName}' must be a non-negative number`), + suffix + ); + assert.strictEqual(databases[DATABASE][tableName], undefined, suffix); + } }); it("leaves an existing table's settings alone when a peer definition differs", async function () { From 79c5031ca7e7d94b43cbf7dd956bff40af3ca7c8 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Tue, 10 Nov 2026 23:19:51 -0700 Subject: [PATCH 05/11] Keep the cleanup scan for rows a cleared TTL stamped across restarts and maintenance pauses Rows written under a TTL keep their expiry after the TTL is cleared, and only a cleanup scan removes them. A declaration that clears a saved expiration or eviction now also marks the catalog row `expiryScan`; a table loaded from such a row keeps the daily scan a table without a TTL would not arm, and so does a class that cleared a scheduled TTL when its maintenance resumes or the main thread becomes worker 0. Co-Authored-By: Claude Opus 5.5 Dispatch-Task: proposal-kriszyp-bundle-harper-table-expiration-config-5df9b5 --- resources/Table.ts | 12 +++-- resources/databases.ts | 8 +++ .../resources/ttlCleanupInterval.test.js | 54 +++++++++++++++++++ 3 files changed, 69 insertions(+), 5 deletions(-) diff --git a/resources/Table.ts b/resources/Table.ts index 65b5b0ea51..68f1260af7 100644 --- a/resources/Table.ts +++ b/resources/Table.ts @@ -2076,6 +2076,8 @@ export function makeTable(options): TableResourceClass { let disposed = false; // true once a table-level expiration/eviction/scanInterval has armed the periodic cleanup scan at setup let expirationScanScheduled = false; + // rows stamped under a TTL since cleared keep their expiry, so their (daily) scan outlives the TTL + let retainExpiryScan = Boolean(options.expiryScan); // set on the first expiring write so the unscheduled-expiration warning is evaluated at most once per table let expirationWarningChecked = false; let writerResidencyWarned = false; @@ -2817,7 +2819,7 @@ export function makeTable(options): TableResourceClass { * main thread loads every table before it becomes worker 0. */ static _armCleanupScan() { - if (expirationScanScheduled || evictionMs) scheduleCleanup(); + if (expirationScanScheduled || evictionMs || retainExpiryScan) scheduleCleanup(); } static getResidencySlot(id: Id, record?: object): number { return calculateResidencySlot(id, record); @@ -4003,11 +4005,11 @@ export function makeTable(options): TableResourceClass { recordScanInterval = (scanIntervalSeconds ?? 0) * 1000; } if (declaredHere && !preserveLoadedConfiguration) ttlDeclaredBySchema = true; - // rows written under a TTL this call clears keep their expiry, so they still need a scan const scanWasScheduled = !preserveLoadedConfiguration && (expirationScanScheduled || cleanupTimer); if (!preserveLoadedConfiguration) { expirationScanScheduled = Boolean(recordScanInterval || expirationMs || evictionMs); cleanupInterval = recordScanInterval || (expirationMs + evictionMs) / 4 || DEFAULT_CLEANUP_INTERVAL; + if (scanWasScheduled && !expirationScanScheduled) retainExpiryScan = true; } if ( replicaTableKnown && @@ -4018,7 +4020,7 @@ export function makeTable(options): TableResourceClass { } else if (replicaEvictionArmed) armReplicaEviction(); // Re-evaluate an existing table-level scan after an ownership-only declaration, but do not // create the default daily cleanup timer for a table that has only an @expiresAt field. - if (expirationScanScheduled || evictionMs || scanWasScheduled) scheduleCleanup(); + if (expirationScanScheduled || evictionMs || retainExpiryScan) scheduleCleanup(); // @expiresAt has its own interval rather than the cleanup timer above. Arm it whenever a live // declaration introduces the attribute, including after this application already claimed TTL. if (expiresAtProperty && !recordExpirationInterval) runRecordExpirationEviction(); @@ -12343,7 +12345,7 @@ export function makeTable(options): TableResourceClass { if (disposed || !maintenanceClosed) return; maintenanceClosed = false; lastCleanupInterval = undefined; - if (expirationScanScheduled || evictionMs) scheduleCleanup(); + if (expirationScanScheduled || evictionMs || retainExpiryScan) scheduleCleanup(); if (replicaEvictionArmed) scheduleReplicaSweep(); if (expiresAtProperty && !recordExpirationInterval) runRecordExpirationEviction(); } @@ -12438,7 +12440,7 @@ export function makeTable(options): TableResourceClass { } finally { ttlFromLoad = false; } - } + } else if (retainExpiryScan) scheduleCleanup(); if (expiresAtProperty && !recordExpirationInterval) runRecordExpirationEviction(); } catch (error) { TableResource.cleanup(); diff --git a/resources/databases.ts b/resources/databases.ts index 92ad5e0310..cd853fc0c6 100644 --- a/resources/databases.ts +++ b/resources/databases.ts @@ -2396,6 +2396,7 @@ function initStores( replicaExpirationMS: replicaExpirationMsOf(replicaExpiration), replicaRefreshFraction: replicaRefreshFractionOf(replicaRefreshFraction), replicaDeadlines: Boolean(tableDef.replicaDeadlines), + expiryScan: primaryAttribute.expiryScan === true, cacheControl, trackDeletes, tableName, @@ -5902,6 +5903,13 @@ function declareTable(target: TableTarget, tableDefinition: T if (audit) Table.enableAuditing(); updatedPrimaryAttribute.audit = audit; } + // Rows already stamped under a TTL being cleared keep their expiry, and only a cleanup scan removes + // them: a load of a table marked this way keeps the daily scan a table without a TTL would not arm. + if ( + (expiration === 0 && savedTtlSeconds(currentPrimaryAttribute.expiration)) || + (eviction === 0 && savedTtlSeconds(currentPrimaryAttribute.eviction)) + ) + updatedPrimaryAttribute.expiryScan = true; if (expiration) updatedPrimaryAttribute.expiration = expiration; else if (expiration === 0) delete updatedPrimaryAttribute.expiration; if (eviction) updatedPrimaryAttribute.eviction = eviction; diff --git a/unitTests/resources/ttlCleanupInterval.test.js b/unitTests/resources/ttlCleanupInterval.test.js index 2b09d4bfe0..3bd0f8701e 100644 --- a/unitTests/resources/ttlCleanupInterval.test.js +++ b/unitTests/resources/ttlCleanupInterval.test.js @@ -24,6 +24,15 @@ async function timersArmedBy(run, settleMs = 0) { } // the TTLs these tests clear scan every 15 s or less const isDailyCadence = (delay) => delay > 60_000 && delay <= 86_400_000; +// a first daily scan with nothing pending lands on the next whole day since the start of the year +function isNextDailySlot(delay) { + const startOfYear = new Date(); + startOfYear.setMonth(0, 1); + startOfYear.setHours(0, 0, 0, 0); + const now = Date.now(); + const slot = Math.ceil((now - startOfYear.getTime()) / 86_400_000) * 86_400_000 + startOfYear.getTime(); + return Math.abs(delay - (slot - now)) < 5000; +} // The cleanup scan is what removes an expired record: reads return it until then. Its interval has to come // from the table's TTL whichever way the TTL was set. A worker that learns of a table from the catalog (a @@ -311,6 +320,51 @@ describe('TTL cleanup scan interval', () => { } }); + it('keeps a scan for rows a cleared TTL stamped after a restart (#3934)', async () => { + const tableName = 'TtlClearedThenRestarted'; + const attributes = () => [{ name: 'id', isPrimaryKey: true }]; + const Declared = table({ table: tableName, database: databaseName, expiration: 60, attributes: attributes() }); + table({ table: tableName, database: databaseName, expiration: 0, attributes: attributes() }); + if (Declared.dbisDB.committed) await Declared.dbisDB.committed; + const primaryRow = Declared.dbisDB.getSync(`${tableName}/`) ?? Declared.dbisDB.getSync(`${tableName}/id`); + assert.strictEqual(primaryRow.expiryScan, true, JSON.stringify(primaryRow)); + Declared.cleanup(); + delete databases[databaseName][tableName]; + let Restarted; + try { + // loaded with no scan owner, so the only timer below is the one this table arms + setMainIsWorker(false); + resetDatabases(); + Restarted = getDatabases()[databaseName][tableName]; + } finally { + setMainIsWorker(true); + } + try { + assert(!Restarted.expirationMS); + const delays = await timersArmedBy(() => Restarted._armCleanupScan()); + assert(delays.length === 1 && isNextDailySlot(delays[0]), `no daily scan armed: ${delays}`); + } finally { + Restarted.cleanup(); + } + }); + + it('keeps a scan for rows a cleared TTL stamped when maintenance resumes (#3934)', async () => { + const Table = table({ + table: 'TtlClearedThenResumed', + database: databaseName, + attributes: [{ name: 'id', isPrimaryKey: true }], + }); + try { + Table.setTTLExpiration({ expiration: 60 }); + Table.setTTLExpiration({ expiration: 0, eviction: 0 }); + await Table.closeMaintenance(); + const delays = await timersArmedBy(() => Table.resumeMaintenance()); + assert(delays.some(isDailyCadence), `resuming maintenance armed no scan: ${delays}`); + } finally { + Table.cleanup(); + } + }); + it('arms no scan for a table with no TTL', async () => { const Table = table({ table: 'TtlNoneToArm', From 9cc72e9fc0b20201cdb80c578834ef9ecb9aa95a Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Tue, 10 Nov 2026 23:34:02 -0700 Subject: [PATCH 06/11] Adopt a catalog expiryScan mark on reload; drop narrating test comments An existing table class that reloads from a catalog row marked expiryScan now keeps the daily scan too, whichever thread's declaration owns its TTL, so rows another thread stamped under a TTL it since cleared are reclaimed before this thread restarts. Co-Authored-By: Claude Opus 5.5 Dispatch-Task: proposal-kriszyp-bundle-harper-table-expiration-config-5df9b5 --- .../database/table-expiration-config.test.ts | 17 +++++++++-------- resources/Table.ts | 9 +++++++-- resources/databases.ts | 2 +- unitTests/resources/ttlCleanupInterval.test.js | 15 +++++++++++++++ 4 files changed, 32 insertions(+), 11 deletions(-) diff --git a/integrationTests/database/table-expiration-config.test.ts b/integrationTests/database/table-expiration-config.test.ts index 5343fdd98d..189f3276a0 100644 --- a/integrationTests/database/table-expiration-config.test.ts +++ b/integrationTests/database/table-expiration-config.test.ts @@ -41,6 +41,14 @@ interface OperationResult { body: any; } +function parseIfJson(text: string): any { + try { + return JSON.parse(text); + } catch { + return text; + } +} + async function operation(ctx: ContextWithHarper, body: Record): Promise { const response = await fetch(ctx.harper.operationsAPIURL, { method: 'POST', @@ -51,13 +59,7 @@ async function operation(ctx: ContextWithHarper, body: Record): body: JSON.stringify(body), }); const text = await response.text(); - let parsed: any = text; - try { - parsed = JSON.parse(text); - } catch { - // not JSON - } - return { status: response.status, body: parsed }; + return { status: response.status, body: parseIfJson(text) }; } async function ok200(ctx: ContextWithHarper, body: Record) { @@ -170,7 +172,6 @@ async function assertRowsKept(ctx: ContextWithHarper, table: string, why: string } } -// Every check after a restart runs, so a failure reports all of them, not only the first. function softChecks() { const failures: string[] = []; return { diff --git a/resources/Table.ts b/resources/Table.ts index 68f1260af7..e3048ff53c 100644 --- a/resources/Table.ts +++ b/resources/Table.ts @@ -1225,7 +1225,7 @@ interface TableResourceClass { getResidencyById: (id: Id) => number | void; readonly residency: Readonly | undefined; _setResidencyPolicy(policy: ResidencyPolicy): void; - _adoptCatalogExpiration(expiration?: number, eviction?: number): void; + _adoptCatalogExpiration(expiration?: number, eviction?: number, expiryScan?: boolean): void; _armCleanupScan(): void; getResidencySlot(id: Id, record?: object): number; getRecordResidency(entry: Partial): readonly string[] | undefined; @@ -2802,7 +2802,12 @@ export function makeTable(options): TableResourceClass { * load would. A TTL that this thread's own schema declaration or application code set is kept, as it is * across a restart, which re-applies it over the catalog. */ - static _adoptCatalogExpiration(expiration?: number, eviction?: number) { + static _adoptCatalogExpiration(expiration?: number, eviction?: number, expiryScan?: boolean) { + // rows another thread stamped under a TTL it since cleared need the scan whoever owns this thread's TTL + if (expiryScan && !retainExpiryScan) { + retainExpiryScan = true; + scheduleCleanup(); + } if (ttlConfiguredByApplication || ttlDeclaredBySchema) return; expiration ||= 0; eviction ||= 0; diff --git a/resources/databases.ts b/resources/databases.ts index cd853fc0c6..fdc303b0c3 100644 --- a/resources/databases.ts +++ b/resources/databases.ts @@ -2376,7 +2376,7 @@ function initStores( if (!destination) requestComputedIndexRepair(table); if (replicaExpiration !== undefined || replicaRefreshFraction !== undefined) table.setTTLExpiration({ replicaExpiration, replicaRefreshFraction, fromSchema: true }); - table._adoptCatalogExpiration(expiration, eviction); + table._adoptCatalogExpiration(expiration, eviction, primaryAttribute.expiryScan === true); } else { if (recreateForTableIdChange) table.cleanup(); table = setTable( diff --git a/unitTests/resources/ttlCleanupInterval.test.js b/unitTests/resources/ttlCleanupInterval.test.js index 3bd0f8701e..c05c64f708 100644 --- a/unitTests/resources/ttlCleanupInterval.test.js +++ b/unitTests/resources/ttlCleanupInterval.test.js @@ -348,6 +348,21 @@ describe('TTL cleanup scan interval', () => { } }); + it('a loaded class arms the scan when a reload finds the catalog marked expiryScan', async () => { + const Declared = table({ + table: 'TtlReloadExpiryScan', + database: databaseName, + attributes: [{ name: 'id', isPrimaryKey: true }], + }); + const Hydrated = hydrate(Declared, 'TtlReloadExpiryScan', {}); + try { + const delays = await timersArmedBy(() => Hydrated._adoptCatalogExpiration(undefined, undefined, true)); + assert(delays.length === 1 && isNextDailySlot(delays[0]), `no daily scan armed: ${delays}`); + } finally { + Hydrated.cleanup(); + } + }); + it('keeps a scan for rows a cleared TTL stamped when maintenance resumes (#3934)', async () => { const Table = table({ table: 'TtlClearedThenResumed', From 695e719b93cadd2483caa0b6b19381af46b434bc Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Wed, 11 Nov 2026 00:09:18 -0700 Subject: [PATCH 07/11] Skip the worker-pinning TTL tests on Windows and Bun They reach a chosen worker by retrying on fresh connections, which needs connections spread across workers: Windows serves HTTP from one worker and Bun does not spread them, so CI's Windows and Bun legs never reached the scan owner. The same skip as sealed-redeclare-tracked-update; the negative-TTL, declaring-thread and threads: 0 suites still run there. Co-Authored-By: Claude Opus 5.5 Dispatch-Task: proposal-kriszyp-bundle-harper-table-expiration-config-5df9b5 --- .../database/table-expiration-config.test.ts | 67 +++++++++++-------- 1 file changed, 39 insertions(+), 28 deletions(-) diff --git a/integrationTests/database/table-expiration-config.test.ts b/integrationTests/database/table-expiration-config.test.ts index 189f3276a0..b0bdfd1b96 100644 --- a/integrationTests/database/table-expiration-config.test.ts +++ b/integrationTests/database/table-expiration-config.test.ts @@ -23,6 +23,9 @@ import { setTimeout as sleep } from 'node:timers/promises'; import { startHarper, killHarper, teardownHarper, type ContextWithHarper } from '@harperfast/integration-testing'; const FIXTURE = 'table-expiration-config'; +// A test reaches a chosen worker by retrying on fresh connections, which needs connections spread across +// workers: Windows serves HTTP from one worker, and Bun does not spread them. +const skipWorkerPinning = process.platform === 'win32' || process.env.HARPER_RUNTIME === 'bun'; const NEGATIVE_FIXTURE = 'table-expiration-config-negative'; const DATABASE = 'ttl_config'; const EXPIRATION = 2; @@ -299,39 +302,47 @@ for (const engine of ['rocksdb', 'lmdb']) { await waitForReclaim(ctx, 'Declared', Date.now() + (EXPIRATION + EVICTION) * 1000, 'a schema-declared TTL'); }); - test('an eviction-only TTL adopted from the catalog reclaims rows (#3935)', { timeout: 120_000 }, async () => { - const table = 'AdoptedEviction'; - const { workerIndex } = await untilServed(ctx, { op: 'declare', table, avoidWorker: OWNER }); - // loaded before it gains a TTL, so the owner adopts the TTL instead of loading the table fresh with it - await waitForWorkerTtl(ctx, table, OWNER, { expirationMS: 0, evictionMS: 0 }); - await untilServed(ctx, { op: 'declare', table, eviction: EVICTION, onWorker: workerIndex }); - await waitForWorkerTtl(ctx, table, OWNER, { expirationMS: 0, evictionMS: EVICTION * 1000 }); - await untilServed(ctx, { op: 'write', table, count: ROWS, expiresInMs: 500 }); - await waitForReclaim(ctx, table, Date.now() + 500 + EVICTION * 1000, 'an eviction-only TTL the owner adopted'); - }); + test( + 'an eviction-only TTL adopted from the catalog reclaims rows (#3935)', + { timeout: 120_000, skip: skipWorkerPinning }, + async () => { + const table = 'AdoptedEviction'; + const { workerIndex } = await untilServed(ctx, { op: 'declare', table, avoidWorker: OWNER }); + // loaded before it gains a TTL, so the owner adopts the TTL instead of loading the table fresh with it + await waitForWorkerTtl(ctx, table, OWNER, { expirationMS: 0, evictionMS: 0 }); + await untilServed(ctx, { op: 'declare', table, eviction: EVICTION, onWorker: workerIndex }); + await waitForWorkerTtl(ctx, table, OWNER, { expirationMS: 0, evictionMS: EVICTION * 1000 }); + await untilServed(ctx, { op: 'write', table, count: ROWS, expiresInMs: 500 }); + await waitForReclaim(ctx, table, Date.now() + 500 + EVICTION * 1000, 'an eviction-only TTL the owner adopted'); + } + ); // both declarations on one worker: a worker keeps a TTL its own declaration set, as a restart re-applies it - test('a re-declaration without a TTL clears it on every worker (#3934)', { timeout: 120_000 }, async () => { - const table = 'AdoptedClear'; - const declared = { expirationMS: EXPIRATION * 1000, evictionMS: EVICTION * 1000 }; - const { workerIndex } = await untilServed(ctx, { - op: 'declare', - table, - expiration: EXPIRATION, - eviction: EVICTION, - avoidWorker: OWNER, - }); - for (let worker = 0; worker < THREADS; worker++) await waitForWorkerTtl(ctx, table, worker, declared); - await untilServed(ctx, { op: 'declare', table, onWorker: workerIndex }); - const checks = softChecks(); - for (let worker = 0; worker < THREADS; worker++) - await checks.check(() => waitForWorkerTtl(ctx, table, worker, { expirationMS: 0, evictionMS: 0 })); - checks.assertNone(); - }); + test( + 'a re-declaration without a TTL clears it on every worker (#3934)', + { timeout: 120_000, skip: skipWorkerPinning }, + async () => { + const table = 'AdoptedClear'; + const declared = { expirationMS: EXPIRATION * 1000, evictionMS: EVICTION * 1000 }; + const { workerIndex } = await untilServed(ctx, { + op: 'declare', + table, + expiration: EXPIRATION, + eviction: EVICTION, + avoidWorker: OWNER, + }); + for (let worker = 0; worker < THREADS; worker++) await waitForWorkerTtl(ctx, table, worker, declared); + await untilServed(ctx, { op: 'declare', table, onWorker: workerIndex }); + const checks = softChecks(); + for (let worker = 0; worker < THREADS; worker++) + await checks.check(() => waitForWorkerTtl(ctx, table, worker, { expirationMS: 0, evictionMS: 0 })); + checks.assertNone(); + } + ); test( 'after a restart: a runtime table is reclaimed without a write, and a removed @table TTL stays removed (#3372, #3934)', - { timeout: 180_000 }, + { timeout: 180_000, skip: skipWorkerPinning }, async () => { const checks = softChecks(); await createRuntimeTtl(ctx); From 663803eeaca6b0be2438fff7467953fc05ac9bb8 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Wed, 11 Nov 2026 05:15:53 -0700 Subject: [PATCH 08/11] Keep an application-set TTL through a schema without one on an isolated owner On an isolated application's thread, ttlConfiguredByApplication meant both "application code set this TTL" and "this thread's schema claimed the scan", so a @table with no expiration cleared a TTL that sourcedFrom options had set (QA-5424). Split them: ttlOwnedByThread keeps the scan ownership latch unchanged, and applicationTtl records which part of the TTL in effect application code set. A declaration with no TTL keeps that part, isolated or not, and clears what a schema authored: a schema TTL that replaced an application's, and the eviction and scanInterval a number-form call left in place. Co-Authored-By: Claude Opus 5.5 Dispatch-Task: harper-4250-f1241-ttl-flag-split --- resources/Table.ts | 35 +++++++------ .../tableExpirationPersistence.test.js | 50 +++++++++++++++++++ .../resources/ttlCleanupInterval.test.js | 34 ++++++++++++- 3 files changed, 104 insertions(+), 15 deletions(-) diff --git a/resources/Table.ts b/resources/Table.ts index e3048ff53c..f1150de9dd 100644 --- a/resources/Table.ts +++ b/resources/Table.ts @@ -2000,7 +2000,10 @@ export function makeTable(options): TableResourceClass { // Set when the TTL exists only on this thread: either application code configured it at runtime, or // an isolated application's schema was declared here. Hydrating persisted metadata does not set it: // dedicated workers open unrelated shared tables too, whose scan remains owned by the pool. - let ttlConfiguredByApplication = false; + let ttlOwnedByThread = false; + // Which part of the TTL in effect application code set, rather than a schema: a declaration with no + // TTL keeps that part and clears the rest. The number form sets only the expiration. + let applicationTtl: 'none' | 'expiration' | 'all' = 'none'; let ttlDeclaredBySchema = false; let ttlFromLoad = false; // true only around a catalog load's setTTLExpiration call evictionMs ??= 0; @@ -2808,7 +2811,7 @@ export function makeTable(options): TableResourceClass { retainExpiryScan = true; scheduleCleanup(); } - if (ttlConfiguredByApplication || ttlDeclaredBySchema) return; + if (ttlOwnedByThread || ttlDeclaredBySchema) return; expiration ||= 0; eviction ||= 0; if (expiration * 1000 === (expirationMs || 0) && eviction * 1000 === evictionMs) return; @@ -3986,29 +3989,33 @@ export function makeTable(options): TableResourceClass { if (validatedRefreshFraction !== undefined) replicaRefreshFraction = validatedRefreshFraction; const declaredHere = !byNumber && opts.fromSchema; const isolatedApplicationOwner = declaredHere && opts.isolatedApplicationOwner; + // a schema without a TTL does not clear one this thread's application code configured + const keepApplicationTtl = + declaredHere && applicationTtl !== 'none' && !expirationSeconds && !evictionSeconds && !scanIntervalSeconds; const preserveLoadedConfiguration = !byNumber && (declaredHere || opts.replicaExpiration !== undefined || opts.replicaRefreshFraction !== undefined) && ((opts.expiration === undefined && opts.eviction === undefined && opts.scanInterval === undefined) || - // a schema without a TTL does not clear one this thread's application code configured - (declaredHere && - ttlConfiguredByApplication && - !isolatedApplicationOwner && - !expirationSeconds && - !evictionSeconds && - !scanIntervalSeconds)); - if (((!ttlFromLoad && !declaredHere) || isolatedApplicationOwner) && !ttlConfiguredByApplication) { - ttlConfiguredByApplication = true; + (keepApplicationTtl && applicationTtl === 'all')); + if (((!ttlFromLoad && !declaredHere) || isolatedApplicationOwner) && !ttlOwnedByThread) { + ttlOwnedByThread = true; // the scan owner may have changed with this: re-evaluate even if the interval did not lastCleanupInterval = undefined; } if (byNumber) { expirationMs = expirationSeconds * 1000; } else if (!preserveLoadedConfiguration) { - expirationMs = (expirationSeconds ?? 0) * 1000; + // kept here means the application set only the expiration: the rest is the schema's to clear + if (!keepApplicationTtl) expirationMs = (expirationSeconds ?? 0) * 1000; evictionMs = (evictionSeconds ?? 0) * 1000; recordScanInterval = (scanIntervalSeconds ?? 0) * 1000; } + if (!preserveLoadedConfiguration && !ttlFromLoad) { + // kept, the application's expiration is all that remains + if (keepApplicationTtl || (!declaredHere && !byNumber)) applicationTtl = 'all'; + else if (declaredHere) applicationTtl = 'none'; + else if (applicationTtl === 'none') applicationTtl = 'expiration'; + } if (declaredHere && !preserveLoadedConfiguration) ttlDeclaredBySchema = true; const scanWasScheduled = !preserveLoadedConfiguration && (expirationScanScheduled || cleanupTimer); if (!preserveLoadedConfiguration) { @@ -14610,7 +14617,7 @@ export function makeTable(options): TableResourceClass { function runRecordExpirationEviction() { if (isReadOnlyMode()) return; // Periodically evict expired records, searching for records who expiresAt timestamp is before now - if (ownsStoreExpiration(primaryStore.path) || (ttlConfiguredByApplication && isDedicatedWorker())) { + if (ownsStoreExpiration(primaryStore.path) || (ttlOwnedByThread && isDedicatedWorker())) { // we want to run the pruning of expired records on only one thread so we don't have conflicts in evicting recordExpirationInterval = setInterval(() => { // go through each database and table and then search for expired entries @@ -15228,7 +15235,7 @@ export function makeTable(options): TableResourceClass { )); } function ownsCleanup(): boolean { - return ownsStoreMaintenance(primaryStore.path) || (ttlConfiguredByApplication && isDedicatedWorker()); + return ownsStoreMaintenance(primaryStore.path) || (ttlOwnedByThread && isDedicatedWorker()); } // an already-armed thread may own the timer only now (the main thread under threads: 0 loads // tables before it is worker 0), so the flag and the timer are settled on every wake diff --git a/unitTests/resources/tableExpirationPersistence.test.js b/unitTests/resources/tableExpirationPersistence.test.js index 73ace9327a..d878b0351d 100644 --- a/unitTests/resources/tableExpirationPersistence.test.js +++ b/unitTests/resources/tableExpirationPersistence.test.js @@ -356,6 +356,56 @@ describe('table-level expiration persistence (harper-pro#1179)', () => { assert.strictEqual(Configured.evictionMS, 10000); }); + it('keeps a TTL application code set through a re-declaration with none, isolated or not (QA-5424)', async function () { + for (const isolatedApplicationOwner of [true, false]) { + const tableName = `ExpiryApplicationRedeclared${isolatedApplicationOwner ? 'Isolated' : 'Shared'}`; + const Configured = keyed(tableName, { expiration: null, eviction: null, isolatedApplicationOwner }); + Configured.setTTLExpiration({ expiration: 30, eviction: 10 }); + keyed(tableName, { expiration: null, eviction: null, isolatedApplicationOwner }); + assert.strictEqual(Configured.expirationMS, 30000, `isolatedApplicationOwner: ${isolatedApplicationOwner}`); + assert.strictEqual(Configured.evictionMS, 10000, `isolatedApplicationOwner: ${isolatedApplicationOwner}`); + await commitCatalog(Configured); + assert.deepStrictEqual(catalogExpiration(Configured), {}, 'an application TTL is never saved'); + } + }); + + it('keeps only the expiration a number-form call set over a schema TTL the schema then drops', async function () { + const tableName = 'ExpiryNumberFormOverSchema'; + const Declared = keyed(tableName, { expiration: 60, eviction: 3600, scanInterval: 3600 }); + Declared.setTTLExpiration(30); + keyed(tableName, { expiration: null, eviction: null }); + assert.strictEqual(Declared.expirationMS, 30000); + assert.strictEqual(Declared.evictionMS, 0, "the eviction was the schema's"); + keyed(tableName, { expiration: null, eviction: null }); + assert.strictEqual(Declared.expirationMS, 30000, 'a further declaration without a TTL keeps it'); + }); + + it("clears a TTL an isolated application's schema declared when it declares none", async function () { + const tableName = 'ExpiryIsolatedSchemaCleared'; + const Declared = keyed(tableName, { expiration: 600, eviction: 300, isolatedApplicationOwner: true }); + assert.strictEqual(Declared.expirationMS, 600000); + keyed(tableName, { expiration: null, eviction: null, isolatedApplicationOwner: true }); + await commitCatalog(Declared); + assert.deepStrictEqual(catalogExpiration(Declared), {}); + assert(!Declared.expirationMS, `this thread still applies ${Declared.expirationMS}ms`); + assert.strictEqual(Declared.evictionMS, 0); + }); + + it('clears a schema TTL that replaced the one application code set, when the schema drops it', async function () { + for (const isolatedApplicationOwner of [true, false]) { + const tableName = `ExpirySchemaReplacedApplication${isolatedApplicationOwner ? 'Isolated' : 'Shared'}`; + const Configured = keyed(tableName, { isolatedApplicationOwner }); + Configured.setTTLExpiration({ expiration: 30 }); + keyed(tableName, { expiration: 120, isolatedApplicationOwner }); + assert.strictEqual(Configured.expirationMS, 120000); + keyed(tableName, { expiration: null, eviction: null, isolatedApplicationOwner }); + assert( + !Configured.expirationMS, + `isolatedApplicationOwner ${isolatedApplicationOwner} kept ${Configured.expirationMS}ms` + ); + } + }); + it('loads a saved value no declaration could save now as absent', async function () { const tableName = 'ExpiryLegacyInvalid'; const Legacy = keyed(tableName); diff --git a/unitTests/resources/ttlCleanupInterval.test.js b/unitTests/resources/ttlCleanupInterval.test.js index c05c64f708..3a39e58dfc 100644 --- a/unitTests/resources/ttlCleanupInterval.test.js +++ b/unitTests/resources/ttlCleanupInterval.test.js @@ -1,7 +1,14 @@ require('../testUtils'); const assert = require('assert'); const { setupTestDBPath } = require('../testUtils'); -const { table, databases, getDatabases, resetDatabases, armTableCleanupScans } = require('#src/resources/databases'); +const { + table, + databases, + getDatabases, + resetDatabases, + armTableCleanupScans, + scopedTableFactory, +} = require('#src/resources/databases'); const { makeTable } = require('#src/resources/Table'); const { setMainIsWorker } = require('#src/server/threads/manageThreads'); const { waitFor } = require('../waitFor'); @@ -127,6 +134,31 @@ describe('TTL cleanup scan interval', () => { } }); + it("an isolated application's schema without a TTL leaves the scan for one application code set (QA-5424)", async () => { + const definition = { + table: 'TtlIsolatedApplicationKept', + database: databaseName, + expiration: null, + eviction: null, + attributes: [{ name: 'id', isPrimaryKey: true }], + }; + // the table as a previous start left it; application code (jsResource) then loads before the schema + const Table = table({ table: definition.table, database: databaseName, attributes: definition.attributes }); + const isolatedTable = scopedTableFactory(undefined, true); + try { + Table.sourcedFrom({ get: (id) => ({ id }) }, { expiration: 0.05 }); + isolatedTable(definition); + isolatedTable(definition); + assert.strictEqual(Table.expirationMS, 50); + await Table.put(1, { id: 1 }); + await Table.primaryStore.committed; + assert(stored(Table, 1)); + await waitFor(() => !stored(Table, 1), { timeout: 5000, message: 'the expired record was never evicted' }); + } finally { + Table.cleanup(); + } + }); + it('the number form keeps a configured scanInterval', () => { const Table = table({ table: 'TtlScanInterval', From a7307459d3f213a2f2c86d7daa25443b206b2820 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Wed, 11 Nov 2026 05:51:29 -0700 Subject: [PATCH 09/11] Give the TTL persistence tests their own database; say a schema 0 keeps an application TTL The persistence suite created its tables in the configured test database, which per-file teardown keeps, so every later file reopened them. The QA-5424 case uses a 500 ms TTL so a slow commit cannot evict the row before the presence check. The schema reference now says a schema that sets a TTL to 0 leaves an application's in place, as one that omits it does. Co-Authored-By: Claude Opus 5.5 Dispatch-Task: harper-4250-f1241-ttl-flag-split --- docs/reference/database/schema.md | 2 +- resources/Table.ts | 1 - unitTests/resources/tableExpirationPersistence.test.js | 3 ++- unitTests/resources/ttlCleanupInterval.test.js | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/reference/database/schema.md b/docs/reference/database/schema.md index 1d20254b14..62cf232078 100644 --- a/docs/reference/database/schema.md +++ b/docs/reference/database/schema.md @@ -109,7 +109,7 @@ type WeatherCache @table(expiration: 300, eviction: 3300, scanInterval: 600) { **Changing or removing them** -`expiration` and `eviction` are saved with the table, and a restart applies what the schema last declared. `@table` (and `defineTable`) sets each of the two that it names and clears each one it omits, including a value `create_table` saved; setting one to `0` clears it too. Without `expiration`, records written afterwards no longer expire; without `eviction`, they are removed as soon as they expire. Records already written keep the expiry they were given. Before v5.4.0 an omitted value stayed in effect until the table was dropped. A TTL that application code sets at runtime (for example through `sourcedFrom` options) is not cleared by a schema that declares none. +`expiration` and `eviction` are saved with the table, and a restart applies what the schema last declared. `@table` (and `defineTable`) sets each of the two that it names and clears each one it omits, including a value `create_table` saved; setting one to `0` clears it too. Without `expiration`, records written afterwards no longer expire; without `eviction`, they are removed as soon as they expire. Records already written keep the expiry they were given. Before v5.4.0 an omitted value stayed in effect until the table was dropped. A TTL that application code sets at runtime (for example through `sourcedFrom` options) is not cleared by a schema that declares none or sets it to `0`; a TTL the schema itself set later is. Each value must be a number of seconds, zero or more. A negative or non-numeric value fails the declaration with an error naming the table and the argument, and the table is left unchanged. diff --git a/resources/Table.ts b/resources/Table.ts index f1150de9dd..49a4aeb1eb 100644 --- a/resources/Table.ts +++ b/resources/Table.ts @@ -4011,7 +4011,6 @@ export function makeTable(options): TableResourceClass { recordScanInterval = (scanIntervalSeconds ?? 0) * 1000; } if (!preserveLoadedConfiguration && !ttlFromLoad) { - // kept, the application's expiration is all that remains if (keepApplicationTtl || (!declaredHere && !byNumber)) applicationTtl = 'all'; else if (declaredHere) applicationTtl = 'none'; else if (applicationTtl === 'none') applicationTtl = 'expiration'; diff --git a/unitTests/resources/tableExpirationPersistence.test.js b/unitTests/resources/tableExpirationPersistence.test.js index d878b0351d..16c544ed1b 100644 --- a/unitTests/resources/tableExpirationPersistence.test.js +++ b/unitTests/resources/tableExpirationPersistence.test.js @@ -13,7 +13,8 @@ const { const { defineTable, types } = require('#src/resources/defineTable'); const { loadGQLSchema } = require('#src/resources/graphql'); -const DATABASE = 'test'; +// its own database, so per-file teardown drops these tables rather than every later file reopening them +const DATABASE = 'ttl_persistence'; async function commitCatalog(Table) { if (Table.dbisDB.committed) await Table.dbisDB.committed; diff --git a/unitTests/resources/ttlCleanupInterval.test.js b/unitTests/resources/ttlCleanupInterval.test.js index 3a39e58dfc..cff9d435fb 100644 --- a/unitTests/resources/ttlCleanupInterval.test.js +++ b/unitTests/resources/ttlCleanupInterval.test.js @@ -146,10 +146,10 @@ describe('TTL cleanup scan interval', () => { const Table = table({ table: definition.table, database: databaseName, attributes: definition.attributes }); const isolatedTable = scopedTableFactory(undefined, true); try { - Table.sourcedFrom({ get: (id) => ({ id }) }, { expiration: 0.05 }); + Table.sourcedFrom({ get: (id) => ({ id }) }, { expiration: 0.5 }); isolatedTable(definition); isolatedTable(definition); - assert.strictEqual(Table.expirationMS, 50); + assert.strictEqual(Table.expirationMS, 500); await Table.put(1, { id: 1 }); await Table.primaryStore.committed; assert(stored(Table, 1)); From be981fbf53076fc7bf9137ab0edbd391a252e0a2 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Wed, 11 Nov 2026 06:04:52 -0700 Subject: [PATCH 10/11] Keep the TTL scan tests clear of slow commits and midnight; no stale expiresAt warning The promotion test's row expired 100 ms after the put while its declaring class still scanned, so a slow commit evicted it early; it now expires after 1 s and the test waits for that. isDailyCadence rejected a daily timer due within a minute of the next slot. The per-record expiresAt warning no longer says the scan does not survive a restart when a cleared TTL's expiryScan mark keeps it. Co-Authored-By: Claude Opus 5.5 Dispatch-Task: harper-4250-f1241-ttl-flag-split --- resources/Table.ts | 9 ++++++++- server/threads/socketRouter.ts | 4 +++- unitTests/resources/tableExpirationPersistence.test.js | 1 - unitTests/resources/ttlCleanupInterval.test.js | 7 ++++--- 4 files changed, 15 insertions(+), 6 deletions(-) diff --git a/resources/Table.ts b/resources/Table.ts index 49a4aeb1eb..150fc52f56 100644 --- a/resources/Table.ts +++ b/resources/Table.ts @@ -7762,7 +7762,14 @@ export function makeTable(options): TableResourceClass { // Evaluate at most once per table (on the first expiring write); later writes short-circuit on one check. if (!expirationWarningChecked) { expirationWarningChecked = true; - if (!expirationMs && !evictionMs && !expirationScanScheduled && !expiresAtProperty && !hasSourceGet) { + if ( + !expirationMs && + !evictionMs && + !expirationScanScheduled && + !retainExpiryScan && + !expiresAtProperty && + !hasSourceGet + ) { logger.warn?.( `A per-record expiresAt was set on table "${tableName}" which has no table-level expiration/eviction, no expiresAt attribute, and no source; expiration will not be reliably enforced (the eviction scan is only armed best-effort on write and does not survive a restart with no writes). Configure a table-level expiration/eviction or an indexed expiresAt attribute for reliable eviction.` ); diff --git a/server/threads/socketRouter.ts b/server/threads/socketRouter.ts index 0cd0918024..ef4f399a65 100644 --- a/server/threads/socketRouter.ts +++ b/server/threads/socketRouter.ts @@ -109,7 +109,9 @@ export async function startHTTPThreads(threadCount = 2, dynamicThreads?: boolean if (isProcessShuttingDown()) return []; await threadServer.startServers(); if (isProcessShuttingDown()) return []; - // After the components' declarations, so a stale saved TTL never arms a scan that a schema replaces + // After the components' declarations, so a stale saved TTL never arms a scan that a schema replaces. + // Workers need no such pass: they have a worker index before loading components, so each table + // arms its own scan as it loads. armTableCleanupScans(); // startServers() schedules listener startup after loading components; await its cached // batch so a bind failure reaches bin/run.ts and exits non-zero in single-thread mode too. diff --git a/unitTests/resources/tableExpirationPersistence.test.js b/unitTests/resources/tableExpirationPersistence.test.js index 16c544ed1b..335882aed9 100644 --- a/unitTests/resources/tableExpirationPersistence.test.js +++ b/unitTests/resources/tableExpirationPersistence.test.js @@ -13,7 +13,6 @@ const { const { defineTable, types } = require('#src/resources/defineTable'); const { loadGQLSchema } = require('#src/resources/graphql'); -// its own database, so per-file teardown drops these tables rather than every later file reopening them const DATABASE = 'ttl_persistence'; async function commitCatalog(Table) { diff --git a/unitTests/resources/ttlCleanupInterval.test.js b/unitTests/resources/ttlCleanupInterval.test.js index cff9d435fb..c8b9dc2d89 100644 --- a/unitTests/resources/ttlCleanupInterval.test.js +++ b/unitTests/resources/ttlCleanupInterval.test.js @@ -30,7 +30,7 @@ async function timersArmedBy(run, settleMs = 0) { return delays; } // the TTLs these tests clear scan every 15 s or less -const isDailyCadence = (delay) => delay > 60_000 && delay <= 86_400_000; +const isDailyCadence = (delay) => (delay > 15_000 && delay <= 86_400_000) || isNextDailySlot(delay); // a first daily scan with nothing pending lands on the next whole day since the start of the year function isNextDailySlot(delay) { const startOfYear = new Date(); @@ -303,7 +303,8 @@ describe('TTL cleanup scan interval', () => { attributes: [{ name: 'id', isPrimaryKey: true }], }); // a restart: the row outlives this class, whose scan stops with it - await Declared.put(1, { id: 1 }, { expiresAt: Date.now() + 100 }); + const expiresAt = Date.now() + 1000; + await Declared.put(1, { id: 1 }, { expiresAt }); await Declared.primaryStore.committed; Declared.cleanup(); delete databases[databaseName][tableName]; @@ -318,7 +319,7 @@ describe('TTL cleanup scan interval', () => { } try { assert.strictEqual(Loaded.expirationMS, 50); - await sleep(400); + await sleep(Math.max(0, expiresAt + 300 - Date.now())); assert(stored(Loaded, 1), 'a scan ran although no thread owned it at load'); armTableCleanupScans(); await waitFor(() => !stored(Loaded, 1), { timeout: 5000, message: 'the expired record was never evicted' }); From 51a5a77c1ff8e8847a6f1708f1fc1ebcdeb2c4ca Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Wed, 11 Nov 2026 06:19:57 -0700 Subject: [PATCH 11/11] Declare the promotion test's TTL after its row commits; match the daily slot's milliseconds Co-Authored-By: Claude Opus 5.5 Dispatch-Task: harper-4250-f1241-ttl-flag-split --- unitTests/resources/ttlCleanupInterval.test.js | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/unitTests/resources/ttlCleanupInterval.test.js b/unitTests/resources/ttlCleanupInterval.test.js index c8b9dc2d89..5ccc99bf62 100644 --- a/unitTests/resources/ttlCleanupInterval.test.js +++ b/unitTests/resources/ttlCleanupInterval.test.js @@ -35,7 +35,7 @@ const isDailyCadence = (delay) => (delay > 15_000 && delay <= 86_400_000) || isN function isNextDailySlot(delay) { const startOfYear = new Date(); startOfYear.setMonth(0, 1); - startOfYear.setHours(0, 0, 0, 0); + startOfYear.setHours(0, 0, 0); // keeps the milliseconds, as scheduleCleanup does const now = Date.now(); const slot = Math.ceil((now - startOfYear.getTime()) / 86_400_000) * 86_400_000 + startOfYear.getTime(); return Math.abs(delay - (slot - now)) < 5000; @@ -296,16 +296,14 @@ describe('TTL cleanup scan interval', () => { it('a table loaded before the main thread became a worker arms its scan once it is one (#2831)', async () => { const tableName = 'TtlLoadedBeforePromotion'; - const Declared = table({ - table: tableName, - database: databaseName, - expiration: 0.05, - attributes: [{ name: 'id', isPrimaryKey: true }], - }); - // a restart: the row outlives this class, whose scan stops with it + const definition = { table: tableName, database: databaseName, attributes: [{ name: 'id', isPrimaryKey: true }] }; + const Declared = table(definition); + // a restart: the row outlives this class, whose scan stops with it; the TTL comes after the commit, so + // no scan of this class can reach the row const expiresAt = Date.now() + 1000; await Declared.put(1, { id: 1 }, { expiresAt }); await Declared.primaryStore.committed; + table({ ...definition, expiration: 0.05 }); Declared.cleanup(); delete databases[databaseName][tableName]; let Loaded;