diff --git a/dataLayer/DESIGN.md b/dataLayer/DESIGN.md index 6062e930d0..481420d215 100644 --- a/dataLayer/DESIGN.md +++ b/dataLayer/DESIGN.md @@ -42,7 +42,7 @@ Set `audit = true` on the directive's `CreateTableObject`: `createTables()` audi - A node that never runs the owner's code path never declares the rest, and its replicated rows are never removed. - An install on a pre-release of the directive's version never runs it (the `compareVersions` caveat). -Repeating a local `table()` declaration repairs any of these in place, keeping every row, and is a no-op once the shape matches. Storage-layer limits: a thread that only loads the table scans daily, so a spent row can stay a day (reads skip it); under `threads: 0` nothing scans until the first expiring write or declaration after restart; and a dropped attribute's index store stays on disk, since `table()` looks it up by table name. The certificate verification tables follow the same pattern through `security/certificateVerification/verificationTables.ts`; their rows are covered in security/DESIGN.md. +Repeating a local `table()` declaration repairs any of these in place, keeping every row, and is a no-op once the shape matches. Storage-layer limit: a dropped attribute's index store stays on disk, since `table()` looks it up by table name. The certificate verification tables follow the same pattern through `security/certificateVerification/verificationTables.ts`; their rows are covered in security/DESIGN.md. ## RocksDB restore: the restore lock + marker protocol (`dataLayer/restoreMarker.ts`) diff --git a/dataLayer/harperBridge/ResourceBridge.ts b/dataLayer/harperBridge/ResourceBridge.ts index 0dc8da27e0..d014b4e78c 100644 --- a/dataLayer/harperBridge/ResourceBridge.ts +++ b/dataLayer/harperBridge/ResourceBridge.ts @@ -168,6 +168,7 @@ export class ResourceBridge extends BridgeMethods { attributes, schemaDefined, expiration: tableCreateObj.expiration, + eviction: tableCreateObj.eviction, audit: tableCreateObj.audit, }); } diff --git a/dataLayer/schemaDescribe.ts b/dataLayer/schemaDescribe.ts index 909e94e66c..64bf24280e 100644 --- a/dataLayer/schemaDescribe.ts +++ b/dataLayer/schemaDescribe.ts @@ -222,7 +222,7 @@ async function descTable(describeTableObject: any, attrPerms?: any) { db_size, }; if (tableObj.replicate !== undefined) tableResult.replicate = tableObj.replicate; - if (tableObj.expirationMS !== undefined) tableResult.expiration = tableObj.expirationMS / 1000 + 's'; + if (tableObj.expirationMS || tableObj.evictionMS) tableResult.expiration = tableObj.expirationMS / 1000 + 's'; if (tableObj.sealed !== undefined) tableResult.sealed = tableObj.sealed; if (tableObj.cacheControl != null) tableResult.cacheControl = tableObj.cacheControl; if ((tableObj as any).sources?.length > 0) diff --git a/docs/reference/database/schema.md b/docs/reference/database/schema.md index c89f0428e3..62cf232078 100644 --- a/docs/reference/database/schema.md +++ b/docs/reference/database/schema.md @@ -107,6 +107,12 @@ type WeatherCache @table(expiration: 300, eviction: 3300, scanInterval: 600) { } ``` +**Changing or removing them** + +`expiration` and `eviction` are saved with the table, and a restart applies what the schema last declared. `@table` (and `defineTable`) sets each of the two that it names and clears each one it omits, including a value `create_table` saved; setting one to `0` clears it too. Without `expiration`, records written afterwards no longer expire; without `eviction`, they are removed as soon as they expire. Records already written keep the expiry they were given. Before v5.4.0 an omitted value stayed in effect until the table was dropped. A TTL that application code sets at runtime (for example through `sourcedFrom` options) is not cleared by a schema that declares none or sets it to `0`; a TTL the schema itself set later is. + +Each value must be a number of seconds, zero or more. A negative or non-numeric value fails the declaration with an error naming the table and the argument, and the table is left unchanged. + #### How `scanInterval` Determines the Eviction Cycle `scanInterval` determines fixed clock-aligned times when eviction runs. Harper divides the clock into evenly spaced anchors based on the interval, calculated in the server's local timezone. As a result: diff --git a/docs/reference/operations-api/operations.md b/docs/reference/operations-api/operations.md index 6ba366e16f..7520668a2d 100644 --- a/docs/reference/operations-api/operations.md +++ b/docs/reference/operations-api/operations.md @@ -175,7 +175,7 @@ Drops a database and all its tables/records. **Replicates to all cluster nodes b ### `create_table` -Creates a new table. Optional fields: `database` (defaults to `data`), `attributes` (array defining schema), `expiration` (TTL in seconds). +Creates a new table. Optional fields: `database` (defaults to `data`), `attributes` (array defining schema), `expiration` (TTL in seconds), `eviction` (seconds after `expiration` before a record is removed; see [`@table`](../database/schema.md#table)). A negative or non-numeric `expiration` or `eviction` is rejected with a 400 error. ```json { diff --git a/docs/release-notes/v5-lincoln/5.4/removing-a-table-expiration-clears-it.md b/docs/release-notes/v5-lincoln/5.4/removing-a-table-expiration-clears-it.md new file mode 100644 index 0000000000..f4a664eea6 --- /dev/null +++ b/docs/release-notes/v5-lincoln/5.4/removing-a-table-expiration-clears-it.md @@ -0,0 +1,7 @@ +--- +section: 'Storage' +--- + +### Removing a Table's Expiration Clears It + +`@table` (and `defineTable`) now clears an `expiration` or `eviction` it no longer names, or sets to `0`, after a restart and on threads that load the table without declaring it. Before, the old value stayed in effect, so a table kept expiring records after its schema stopped declaring a TTL. If you relied on that, declare the value again. Records already written keep the expiry they were given. A negative or non-numeric `expiration`, `eviction` or `scanInterval` now fails the declaration with an error naming the table and argument, instead of failing the table later; `create_table` accepts `eviction` and rejects a negative or non-numeric `expiration` or `eviction` with a 400. Expired records are now removed on schedule after a restart with `threads.count: 0`, and on a table that has only `eviction`, without waiting for a write. See [`@table`](/reference/v5/database/schema#table). diff --git a/integrationTests/database/table-expiration-config-negative/config.yaml b/integrationTests/database/table-expiration-config-negative/config.yaml new file mode 100644 index 0000000000..bb034a091d --- /dev/null +++ b/integrationTests/database/table-expiration-config-negative/config.yaml @@ -0,0 +1,5 @@ +rest: true +graphqlSchema: + files: 'schema.graphql' + # Shortened from the 30s default so a run that times out instead of failing fast costs seconds. + timeout: 5000 diff --git a/integrationTests/database/table-expiration-config-negative/schema.graphql b/integrationTests/database/table-expiration-config-negative/schema.graphql new file mode 100644 index 0000000000..7dcc9649cc --- /dev/null +++ b/integrationTests/database/table-expiration-config-negative/schema.graphql @@ -0,0 +1,4 @@ +type NegativeTtl @table(database: "ttl_config_bad", expiration: -1) @export { + id: ID @primaryKey + value: String +} diff --git a/integrationTests/database/table-expiration-config.test.ts b/integrationTests/database/table-expiration-config.test.ts new file mode 100644 index 0000000000..b0bdfd1b96 --- /dev/null +++ b/integrationTests/database/table-expiration-config.test.ts @@ -0,0 +1,405 @@ +/** + * A table's TTL configuration on every thread that holds the table: the threads that declare it, the threads + * that only load it from the catalog, and the main thread under `threads: 0`, across restarts, on RocksDB and + * LMDB (harper#1286, #2831, #3372, #3934, #3935). + * + * The cleanup scan runs on the store-maintenance owner, the last worker. A table created by `create_table`, or + * declared through the fixture's TtlControl endpoint on another worker, reaches the owner only through the + * catalog. Under `threads: 0` the main thread loads every table before it becomes worker 0. + * + * Expiration and eviction are 2 s each (8 s on the tables that live across a restart), so the derived scan + * interval is (2 + 2) / 4 = 1 s (4 s). + * + * Reproduction: + * npm run build && npm run test:integration -- "integrationTests/database/table-expiration-config.test.ts" + */ +import { suite, test, before, after } from 'node:test'; +import { ok, strictEqual } from 'node:assert'; +import { request as httpRequest } from 'node:http'; +import { join, resolve } from 'node:path'; +import { cp, mkdtemp, readFile, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { setTimeout as sleep } from 'node:timers/promises'; +import { startHarper, killHarper, teardownHarper, type ContextWithHarper } from '@harperfast/integration-testing'; + +const FIXTURE = 'table-expiration-config'; +// A test reaches a chosen worker by retrying on fresh connections, which needs connections spread across +// workers: Windows serves HTTP from one worker, and Bun does not spread them. +const skipWorkerPinning = process.platform === 'win32' || process.env.HARPER_RUNTIME === 'bun'; +const NEGATIVE_FIXTURE = 'table-expiration-config-negative'; +const DATABASE = 'ttl_config'; +const EXPIRATION = 2; +const EVICTION = 2; +const ROWS = 5; +// Tables that live across a restart use a longer TTL, so their rows only become evictable after the restart and +// only the restarted process can reclaim them. Derived scan interval (8 + 8) / 4 = 4 s. +const RUNTIME_EXPIRATION = 8; +const RUNTIME_EVICTION = 8; +const RECLAIM_MARGIN_MS = 15_000; +// past expiration + eviction plus three scans, so a TTL still in effect would have hidden and reclaimed the rows +const CLEARED_HOLD_MS = (EXPIRATION + EVICTION) * 1000 + 3_000; + +interface OperationResult { + status: number; + body: any; +} + +function parseIfJson(text: string): any { + try { + return JSON.parse(text); + } catch { + return text; + } +} + +async function operation(ctx: ContextWithHarper, body: Record): Promise { + const response = await fetch(ctx.harper.operationsAPIURL, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Authorization': basicAuth(ctx), + }, + body: JSON.stringify(body), + }); + const text = await response.text(); + return { status: response.status, body: parseIfJson(text) }; +} + +async function ok200(ctx: ContextWithHarper, body: Record) { + const result = await operation(ctx, body); + strictEqual(result.status, 200, `${body.operation} failed: ${JSON.stringify(result.body)}`); + return result.body; +} + +// describe_table counts stored entries without a query or GET, so it neither hides nor touches expired rows +async function storedRows(ctx: ContextWithHarper, table: string): Promise { + return (await ok200(ctx, { operation: 'describe_table', database: DATABASE, table })).record_count; +} + +async function waitForReclaim(ctx: ContextWithHarper, table: string, dueAt: number, why: string) { + const deadline = dueAt + RECLAIM_MARGIN_MS; + let count: number; + do { + await sleep(500); + count = await storedRows(ctx, table); + } while (count > 0 && Date.now() < deadline); + strictEqual( + count, + 0, + `${count} rows of ${table} still stored ${RECLAIM_MARGIN_MS / 1000}s after they were due: ${why}` + ); +} + +async function insertRows(ctx: ContextWithHarper, table: string) { + const records = Array.from({ length: ROWS }, (_, i) => ({ id: String(i), value: `v${i}` })); + await ok200(ctx, { operation: 'insert', database: DATABASE, table, records }); +} + +const basicAuth = (ctx: ContextWithHarper) => + `Basic ${Buffer.from(`${ctx.harper.admin.username}:${ctx.harper.admin.password}`).toString('base64')}`; + +// a fresh connection per request, so retries reach different workers +function http( + ctx: ContextWithHarper, + method: string, + path: string, + body?: Record +): Promise<{ status: number; text: string }> { + const payload = body === undefined ? undefined : JSON.stringify(body); + return new Promise((resolveRequest, reject) => { + const req = httpRequest( + new URL(path, ctx.harper.httpURL), + { + method, + agent: false, + headers: { + Authorization: basicAuth(ctx), + ...(payload === undefined + ? {} + : { 'Content-Type': 'application/json', 'Content-Length': Buffer.byteLength(payload) }), + }, + }, + (res) => { + const chunks: Buffer[] = []; + res.on('data', (chunk) => chunks.push(chunk)); + res.on('end', () => resolveRequest({ status: res.statusCode ?? 0, text: Buffer.concat(chunks).toString() })); + } + ); + req.setTimeout(30_000, () => req.destroy(new Error(`${method} ${path}: no response`))); + req.on('error', reject); + req.end(payload); + }); +} + +async function control(ctx: ContextWithHarper, body: Record): Promise { + const { status, text } = await http(ctx, 'POST', '/TtlControl/', body); + if (status !== 200) throw new Error(`TtlControl ${body.op}: ${status} ${text}`); + return JSON.parse(text); +} + +async function untilServed(ctx: ContextWithHarper, body: Record) { + const deadline = Date.now() + 60_000; + for (;;) { + const result = await control(ctx, body); + if (!result.wrongThread) return result; + if (Date.now() > deadline) throw new Error(`TtlControl ${body.op} never reached a worker that could serve it`); + await sleep(10); + } +} + +const workerTtl = (ctx: ContextWithHarper, table: string, worker: number) => + untilServed(ctx, { op: 'ttl', table, onWorker: worker }); + +async function waitForWorkerTtl( + ctx: ContextWithHarper, + table: string, + worker: number, + expected: { expirationMS: number; evictionMS: number } +) { + const deadline = Date.now() + 20_000; + let seen: any; + do { + seen = await workerTtl(ctx, table, worker); + if (seen.loaded && seen.expirationMS === expected.expirationMS && seen.evictionMS === expected.evictionMS) return; + await sleep(200); + } while (Date.now() < deadline); + throw new Error(`worker ${worker} holds ${JSON.stringify(seen)} for ${table}, expected ${JSON.stringify(expected)}`); +} + +async function assertRowsKept(ctx: ContextWithHarper, table: string, why: string) { + await sleep(CLEARED_HOLD_MS); + strictEqual(await storedRows(ctx, table), ROWS, `rows of ${table} were reclaimed on: ${why}`); + for (let i = 0; i < ROWS * 4; i++) { + const { status } = await http(ctx, 'GET', `/${table}/${i % ROWS}`); + strictEqual(status, 200, `GET /${table}/${i % ROWS} was hidden as expired on: ${why}`); + } +} + +function softChecks() { + const failures: string[] = []; + return { + async check(run: () => Promise) { + try { + await run(); + } catch (error) { + failures.push((error as Error).message); + } + }, + assertNone() { + strictEqual(failures.length, 0, `${failures.length} checks failed:\n- ${failures.join('\n- ')}`); + }, + }; +} + +async function stage(fixtures: string[]) { + const dataRootDir = await mkdtemp( + join(process.env.HARPER_INTEGRATION_TEST_INSTALL_PARENT_DIR || tmpdir(), 'harper-integration-test-') + ); + for (const fixture of fixtures) + await cp(resolve(import.meta.dirname, fixture), join(dataRootDir, 'components', fixture), { + recursive: true, + dereference: true, + }); + return dataRootDir; +} + +async function dropDeclaredTtl(ctx: ContextWithHarper) { + const schemaPath = join(ctx.harper.dataRootDir, 'components', FIXTURE, 'schema.graphql'); + const schema = await readFile(schemaPath, 'utf8'); + const cleared = schema.replace(`, expiration: ${EXPIRATION}, eviction: ${EVICTION}`, ''); + ok(cleared !== schema, 'the fixture schema no longer declares the TTL this test removes'); + await writeFile(schemaPath, cleared); +} + +async function createRuntimeTtl(ctx: ContextWithHarper) { + await ok200(ctx, { + operation: 'create_table', + database: DATABASE, + table: 'RuntimeTtl', + primary_key: 'id', + expiration: RUNTIME_EXPIRATION, + eviction: RUNTIME_EVICTION, + }); +} + +async function restartWithoutDeclaredTtl(ctx: ContextWithHarper, options: Record, dueAt: number) { + await killHarper(ctx); + await dropDeclaredTtl(ctx); + await startHarper(ctx, options); + ok(Date.now() < dueAt, 'the restart outlasted the runtime TTL, so it cannot show which process reclaimed the rows'); +} + +async function assertDeclaredTtlRemoved(ctx: ContextWithHarper) { + const described = await ok200(ctx, { operation: 'describe_table', database: DATABASE, table: 'Declared' }); + strictEqual(described.expiration, undefined, 'describe_table (the main thread) still reports the removed expiration'); + await insertRows(ctx, 'Declared'); + await assertRowsKept(ctx, 'Declared', 'a schema whose TTL was removed, after a restart'); +} + +for (const engine of ['rocksdb', 'lmdb']) { + const env = engine === 'lmdb' ? { HARPER_STORAGE_ENGINE: 'lmdb' } : {}; + + suite(`Table TTL configuration with four workers (${engine})`, (ctx: ContextWithHarper) => { + const THREADS = 4; + const OWNER = THREADS - 1; + const options = { config: { threads: { count: THREADS } }, env }; + + before(async () => { + ctx.harper = { dataRootDir: await stage([FIXTURE, NEGATIVE_FIXTURE]) } as any; + await startHarper(ctx, options); + }); + + after(async () => { + await teardownHarper(ctx); + }); + + test('a negative TTL fails where it is declared, naming the table and field (#1286, #3935)', async () => { + const name = `${NEGATIVE_FIXTURE}.graphqlSchema`; + const deadline = Date.now() + 30_000; + let entry: any; + do { + const status = await ok200(ctx, { operation: 'get_status' }); + entry = status.componentStatus?.find((candidate: any) => candidate.name === name); + if (entry?.status === 'error') break; + await sleep(250); + } while (Date.now() < deadline); + strictEqual( + entry?.status, + 'error', + `@table(expiration: -1) did not fail its component: ${JSON.stringify(entry)}` + ); + const reported = JSON.stringify(entry); + ok(!/timed out/i.test(reported), `the component failed on a timeout instead of the TTL: ${reported}`); + ok( + /expiration/.test(reported) && reported.includes('ttl_config_bad.NegativeTtl'), + `the failure does not name the table and field: ${reported}` + ); + + for (const [field, table] of [ + ['eviction', 'NegativeEviction'], + ['expiration', 'NegativeExpiration'], + ]) { + const created = await operation(ctx, { + operation: 'create_table', + database: DATABASE, + table, + primary_key: 'id', + [field]: -1, + }); + strictEqual(created.status, 400, `create_table with ${field}: -1 returned ${JSON.stringify(created)}`); + const error = JSON.stringify(created.body); + ok( + error.includes(field) && error.includes(`${DATABASE}.${table}`), + `create_table does not name the table and field: ${error}` + ); + const described = await operation(ctx, { operation: 'describe_table', database: DATABASE, table }); + ok(described.status !== 200, `a rejected create_table left ${table} behind`); + } + }); + + test('the declaring thread reclaims expired rows on the derived cadence', { timeout: 60_000 }, async () => { + await insertRows(ctx, 'Declared'); + await waitForReclaim(ctx, 'Declared', Date.now() + (EXPIRATION + EVICTION) * 1000, 'a schema-declared TTL'); + }); + + test( + 'an eviction-only TTL adopted from the catalog reclaims rows (#3935)', + { timeout: 120_000, skip: skipWorkerPinning }, + async () => { + const table = 'AdoptedEviction'; + const { workerIndex } = await untilServed(ctx, { op: 'declare', table, avoidWorker: OWNER }); + // loaded before it gains a TTL, so the owner adopts the TTL instead of loading the table fresh with it + await waitForWorkerTtl(ctx, table, OWNER, { expirationMS: 0, evictionMS: 0 }); + await untilServed(ctx, { op: 'declare', table, eviction: EVICTION, onWorker: workerIndex }); + await waitForWorkerTtl(ctx, table, OWNER, { expirationMS: 0, evictionMS: EVICTION * 1000 }); + await untilServed(ctx, { op: 'write', table, count: ROWS, expiresInMs: 500 }); + await waitForReclaim(ctx, table, Date.now() + 500 + EVICTION * 1000, 'an eviction-only TTL the owner adopted'); + } + ); + + // both declarations on one worker: a worker keeps a TTL its own declaration set, as a restart re-applies it + test( + 'a re-declaration without a TTL clears it on every worker (#3934)', + { timeout: 120_000, skip: skipWorkerPinning }, + async () => { + const table = 'AdoptedClear'; + const declared = { expirationMS: EXPIRATION * 1000, evictionMS: EVICTION * 1000 }; + const { workerIndex } = await untilServed(ctx, { + op: 'declare', + table, + expiration: EXPIRATION, + eviction: EVICTION, + avoidWorker: OWNER, + }); + for (let worker = 0; worker < THREADS; worker++) await waitForWorkerTtl(ctx, table, worker, declared); + await untilServed(ctx, { op: 'declare', table, onWorker: workerIndex }); + const checks = softChecks(); + for (let worker = 0; worker < THREADS; worker++) + await checks.check(() => waitForWorkerTtl(ctx, table, worker, { expirationMS: 0, evictionMS: 0 })); + checks.assertNone(); + } + ); + + test( + 'after a restart: a runtime table is reclaimed without a write, and a removed @table TTL stays removed (#3372, #3934)', + { timeout: 180_000, skip: skipWorkerPinning }, + async () => { + const checks = softChecks(); + await createRuntimeTtl(ctx); + await insertRows(ctx, 'RuntimeTtl'); + const dueAt = Date.now() + (RUNTIME_EXPIRATION + RUNTIME_EVICTION) * 1000; + await restartWithoutDeclaredTtl(ctx, options, dueAt); + + await checks.check(() => waitForReclaim(ctx, 'RuntimeTtl', dueAt, 'a runtime-created table, no write')); + for (let worker = 0; worker < THREADS; worker++) + await checks.check(() => waitForWorkerTtl(ctx, 'Declared', worker, { expirationMS: 0, evictionMS: 0 })); + await checks.check(() => assertDeclaredTtlRemoved(ctx)); + checks.assertNone(); + } + ); + }); + + suite(`Table TTL configuration with threads: 0 (${engine})`, (ctx: ContextWithHarper) => { + const options = { config: { threads: { count: 0 } }, env }; + + before(async () => { + ctx.harper = { dataRootDir: await stage([FIXTURE]) } as any; + await startHarper(ctx, options); + }); + + after(async () => { + await teardownHarper(ctx); + }); + + test( + 'after a restart, the main thread reclaims catalog-loaded tables without a write and keeps a removed TTL removed (#2831, #3934, #3935)', + { timeout: 180_000 }, + async () => { + const checks = softChecks(); + strictEqual((await ok200(ctx, { operation: 'get_configuration' })).threads.count, 0); + await createRuntimeTtl(ctx); + await ok200(ctx, { + operation: 'create_table', + database: DATABASE, + table: 'RuntimeEviction', + primary_key: 'id', + eviction: RUNTIME_EVICTION, + }); + await checks.check(() => + waitForWorkerTtl(ctx, 'RuntimeEviction', 0, { expirationMS: 0, evictionMS: RUNTIME_EVICTION * 1000 }) + ); + await insertRows(ctx, 'RuntimeTtl'); + const expiresInMs = RUNTIME_EXPIRATION * 1000; + await untilServed(ctx, { op: 'write', table: 'RuntimeEviction', count: ROWS, expiresInMs }); + const dueAt = Date.now() + (RUNTIME_EXPIRATION + RUNTIME_EVICTION) * 1000; + await restartWithoutDeclaredTtl(ctx, options, dueAt); + + await checks.check(() => waitForReclaim(ctx, 'RuntimeTtl', dueAt, 'a runtime-created table, no write')); + await checks.check(() => waitForReclaim(ctx, 'RuntimeEviction', dueAt, 'an eviction-only runtime table')); + await checks.check(() => waitForWorkerTtl(ctx, 'Declared', 0, { expirationMS: 0, evictionMS: 0 })); + await checks.check(() => assertDeclaredTtlRemoved(ctx)); + checks.assertNone(); + } + ); + }); +} diff --git a/integrationTests/database/table-expiration-config/config.yaml b/integrationTests/database/table-expiration-config/config.yaml new file mode 100644 index 0000000000..41005f90af --- /dev/null +++ b/integrationTests/database/table-expiration-config/config.yaml @@ -0,0 +1,5 @@ +rest: true +graphqlSchema: + files: 'schema.graphql' +jsResource: + files: 'resources.js' diff --git a/integrationTests/database/table-expiration-config/resources.js b/integrationTests/database/table-expiration-config/resources.js new file mode 100644 index 0000000000..479eeaf253 --- /dev/null +++ b/integrationTests/database/table-expiration-config/resources.js @@ -0,0 +1,47 @@ +// POST /TtlControl/ drives table-expiration-config.test.ts: +// { op: 'declare', table, expiration?, eviction?, avoidWorker?, onWorker? } — defineTable on the worker that +// serves the request; a worker that avoidWorker excludes, or onWorker does not name, answers { wrongThread } +// and the test retries on a fresh connection, so the scan owner only ever learns the TTL from the catalog +// { op: 'ttl', table, onWorker } — that worker's view of the table: whether it is loaded, and its TTL in ms +// { op: 'write', table, count, expiresInMs } — rows whose own expiry is expiresInMs from now +import { Resource, databases, defineTable, types } from 'harper'; +import { workerData } from 'node:worker_threads'; + +const DATABASE = 'ttl_config'; +const workerIndex = () => workerData?.workerIndex ?? 0; + +export class TtlControl extends Resource { + static loadAsInstance = false; + async post(_target, body) { + const { op, table } = body; + if (op === 'declare') { + if ( + (body.avoidWorker !== undefined && workerIndex() === body.avoidWorker) || + (body.onWorker !== undefined && workerIndex() !== body.onWorker) + ) + return { wrongThread: true, workerIndex: workerIndex() }; + const options = { database: DATABASE }; + if (body.expiration !== undefined) options.expiration = body.expiration; + if (body.eviction !== undefined) options.eviction = body.eviction; + defineTable(table, { id: types.id.primaryKey, value: types.string.nullable }, options); + return { workerIndex: workerIndex() }; + } + if (op === 'ttl') { + if (workerIndex() !== body.onWorker) return { wrongThread: true, workerIndex: workerIndex() }; + const Table = databases[DATABASE]?.[table]; + return { + workerIndex: workerIndex(), + loaded: Boolean(Table), + expirationMS: Table?.expirationMS || 0, + evictionMS: Table?.evictionMS || 0, + }; + } + if (op === 'write') { + const Table = databases[DATABASE][table]; + const expiresAt = Date.now() + body.expiresInMs; + for (let i = 0; i < body.count; i++) await Table.put(String(i), { id: String(i), value: 'v' }, { expiresAt }); + return { workerIndex: workerIndex(), expiresAt }; + } + throw new Error(`unknown op ${op}`); + } +} diff --git a/integrationTests/database/table-expiration-config/schema.graphql b/integrationTests/database/table-expiration-config/schema.graphql new file mode 100644 index 0000000000..a0347c5115 --- /dev/null +++ b/integrationTests/database/table-expiration-config/schema.graphql @@ -0,0 +1,5 @@ +# The test rewrites this file between restarts to drop the TTL arguments. +type Declared @table(database: "ttl_config", expiration: 2, eviction: 2) @export { + id: ID @primaryKey + value: String +} diff --git a/resources/DESIGN.md b/resources/DESIGN.md index 78e0a7f892..d746c0b529 100644 --- a/resources/DESIGN.md +++ b/resources/DESIGN.md @@ -269,7 +269,7 @@ If this same write (origin and log key) emptied the key, `_writeDelete` writes n ## Cluster-origin table definitions are additive-only -An `origin: 'cluster'` definition is a peer snapshot, so `table()` only adds from it and logs what it discards. Drops and redefinitions never converge via gossip (recover via local `create_attribute` or redeploy). Abandoned index builds still recover (`clusterSchemaMerge.test.js`). A reload keeps a thread's own TTL. +An `origin: 'cluster'` definition is a peer snapshot, so `table()` only adds from it and logs what it discards. Drops and redefinitions never converge via gossip (recover via local `create_attribute` or redeploy). Abandoned index builds still recover (`clusterSchemaMerge.test.js`). A reload keeps a thread's own TTL. `@table` omitting a TTL clears it (`null`); `table()` omitting one keeps it. ## `drop_attribute` never removes the primary key, and a strip keeps each row's version diff --git a/resources/MAP.md b/resources/MAP.md index 4f2cf90865..5533f135af 100644 --- a/resources/MAP.md +++ b/resources/MAP.md @@ -46,7 +46,7 @@ Where things are in `resources/`. The rules live in [DESIGN.md](DESIGN.md): read - response shape (select) → `transformEntryForSelect` in Table.ts; `transformForSelect` in Resource.ts - selects answered from index entries alone → `coveredSelectAttribute` in Table.ts; `coverIndexEntries` in search.ts - sorting → `transformToOrderedSelect` in Table.ts; whether the lead already supplies the order → `scansInAttributeOrder` in search.ts -- record TTL and eviction → `setTTLExpiration`, `_adoptCatalogExpiration`, `evict`, `scheduleCleanup` in Table.ts +- record TTL and eviction → `setTTLExpiration`, `_adoptCatalogExpiration`, `evict`, `scheduleCleanup` in Table.ts; `armTableCleanupScans` in databases.ts - cache miss to source → `getFromSource` in Table.ts - subscriptions and replay → `subscribe`, `eventFromAudit` in Table.ts; `addSubscription` in transactionBroadcast.ts - replication apply loop → `sourcedFrom` in Table.ts diff --git a/resources/Table.ts b/resources/Table.ts index 61de4089e7..150fc52f56 100644 --- a/resources/Table.ts +++ b/resources/Table.ts @@ -811,6 +811,36 @@ export function replicaExpirationMsOf(seconds: unknown): number | undefined { if (!Number.isFinite(ms)) throw new Error('replicaExpiration is too large'); return ms; } +// the scan cadence of a table with no TTL, which an expiring write arms for its per-record expiry +const DEFAULT_CLEANUP_INTERVAL = 86_400_000; +// the latest Date, in seconds: an expiry past it cannot be stamped on a record +const MAX_TTL_SECONDS = 8.64e12; +/** + * A table's `expiration`, `eviction` or `scanInterval` in seconds, as a declaration or `setTTLExpiration` gives + * it: undefined when not given, 0 for `null` (cleared), else a number or a numeric string (a create persisted it + * as given) that is finite and zero or more. + */ +export function ttlSecondsOf(table: string, field: string, value: unknown): number | undefined { + if (value === undefined) return undefined; + if (value === null) return 0; + const seconds = + typeof value === 'number' ? value : typeof value === 'string' && value.trim() !== '' ? Number(value) : NaN; + if (seconds >= 0 && seconds <= MAX_TTL_SECONDS) return seconds; + throw new ClientError( + `${field} of '${table}' must be a non-negative number of seconds, got ${describeTtlValue(value)}` + ); +} +/** A TTL field as the catalog holds it: absent unless a valid positive number of seconds. */ +export function savedTtlSeconds(value: unknown): number | undefined { + const seconds = typeof value === 'number' || typeof value === 'string' ? Number(value) : NaN; + return seconds > 0 && seconds <= MAX_TTL_SECONDS ? seconds : undefined; +} +/** A rejected TTL value for a message: bounded, and never a value that cannot be rendered. */ +export function describeTtlValue(value: unknown): string { + if (typeof value === 'number' || typeof value === 'bigint' || typeof value === 'boolean') return String(value); + if (typeof value === 'string') return JSON.stringify(value.slice(0, 40)); + return value === null ? 'null' : typeof value; +} const TEST_WRITE_KEY_BUFFER = Buffer.allocUnsafeSlow(8192); const MAX_KEY_BYTES = 1978; const EVENT_HIGH_WATER_MARK = 100; @@ -1195,7 +1225,8 @@ interface TableResourceClass { getResidencyById: (id: Id) => number | void; readonly residency: Readonly | undefined; _setResidencyPolicy(policy: ResidencyPolicy): void; - _adoptCatalogExpiration(expiration?: number, eviction?: number): void; + _adoptCatalogExpiration(expiration?: number, eviction?: number, expiryScan?: boolean): void; + _armCleanupScan(): void; getResidencySlot(id: Id, record?: object): number; getRecordResidency(entry: Partial): readonly string[] | undefined; get expirationMS(): any; @@ -1964,13 +1995,15 @@ export function makeTable(options): TableResourceClass { // a store on any thread wakes the sweep when [1] is 0 let replicaSweepState: Int32Array | undefined; let recordScanInterval = 0; - let recordCleanupInterval = 86400000; const replicaRefreshInFlight = new Set(); const replicaRefreshInFlightComposite = new Set(); // Set when the TTL exists only on this thread: either application code configured it at runtime, or // an isolated application's schema was declared here. Hydrating persisted metadata does not set it: // dedicated workers open unrelated shared tables too, whose scan remains owned by the pool. - let ttlConfiguredByApplication = false; + let ttlOwnedByThread = false; + // Which part of the TTL in effect application code set, rather than a schema: a declaration with no + // TTL keeps that part and clears the rest. The number form sets only the expiration. + let applicationTtl: 'none' | 'expiration' | 'all' = 'none'; let ttlDeclaredBySchema = false; let ttlFromLoad = false; // true only around a catalog load's setTTLExpiration call evictionMs ??= 0; @@ -2036,7 +2069,7 @@ export function makeTable(options): TableResourceClass { let prefetchCallbacks = []; let untilNextPrefetch = 1; let nonPrefetchSequence = 2; - let cleanupInterval = 86400000; + let cleanupInterval = DEFAULT_CLEANUP_INTERVAL; let cleanupPriority = 0; let lastCleanupInterval: number | undefined; let cleanupTimer: NodeJS.Timeout | undefined; @@ -2046,6 +2079,8 @@ export function makeTable(options): TableResourceClass { let disposed = false; // true once a table-level expiration/eviction/scanInterval has armed the periodic cleanup scan at setup let expirationScanScheduled = false; + // rows stamped under a TTL since cleared keep their expiry, so their (daily) scan outlives the TTL + let retainExpiryScan = Boolean(options.expiryScan); // set on the first expiring write so the unscheduled-expiration warning is evaluated at most once per table let expirationWarningChecked = false; let writerResidencyWarned = false; @@ -2770,24 +2805,30 @@ export function makeTable(options): TableResourceClass { * load would. A TTL that this thread's own schema declaration or application code set is kept, as it is * across a restart, which re-applies it over the catalog. */ - static _adoptCatalogExpiration(expiration?: number, eviction?: number) { - if (ttlConfiguredByApplication || ttlDeclaredBySchema) return; - const loadedExpirationMs = expiration && expiration > 0 ? expiration * 1000 : 0; - const loadedEvictionMs = eviction && eviction > 0 ? eviction * 1000 : 0; - if (!loadedExpirationMs && !loadedEvictionMs) return; - if (loadedExpirationMs === expirationMs && loadedEvictionMs === evictionMs) return; - evictionMs = loadedEvictionMs; - if (!loadedExpirationMs) { - expirationMs = 0; - return; - } + static _adoptCatalogExpiration(expiration?: number, eviction?: number, expiryScan?: boolean) { + // rows another thread stamped under a TTL it since cleared need the scan whoever owns this thread's TTL + if (expiryScan && !retainExpiryScan) { + retainExpiryScan = true; + scheduleCleanup(); + } + if (ttlOwnedByThread || ttlDeclaredBySchema) return; + expiration ||= 0; + eviction ||= 0; + if (expiration * 1000 === (expirationMs || 0) && eviction * 1000 === evictionMs) return; ttlFromLoad = true; try { - TableResource.setTTLExpiration(loadedExpirationMs / 1000); + TableResource.setTTLExpiration({ expiration, eviction }); } finally { ttlFromLoad = false; } } + /** + * Arm the cleanup scan of a table this thread loaded before it could own the scan: under `threads: 0` the + * main thread loads every table before it becomes worker 0. + */ + static _armCleanupScan() { + if (expirationScanScheduled || evictionMs || retainExpiryScan) scheduleCleanup(); + } static getResidencySlot(id: Id, record?: object): number { return calculateResidencySlot(id, record); } @@ -3913,9 +3954,11 @@ export function makeTable(options): TableResourceClass { * @param opts Time in seconds until records expire, or an options object with `expiration`, `eviction`, * and `scanInterval` (all in seconds, all optional). Number form preserves any previously configured * eviction/scanInterval; object form replaces all three, unless it names only the replica options. An internal schema ownership-only call with - * none of those values preserves the settings already loaded from the catalog. + * none of those values preserves the settings already loaded from the catalog, and so does an internal + * schema call with no TTL on a thread whose application code configured one. * Either form then scans for expired records every `scanInterval`, or a quarter of expiration + eviction - * when none is configured. + * when none is configured, or daily (for per-record expiry) when the table has no TTL. A rejected value + * changes nothing. */ static setTTLExpiration( opts: @@ -3932,42 +3975,52 @@ export function makeTable(options): TableResourceClass { ) { if (opts == null || (typeof opts !== 'number' && typeof opts !== 'object')) throw new Error('Invalid expiration value type'); + const qualifiedName = `${databaseName}.${tableName}`; + const byNumber = typeof opts === 'number'; + const expirationSeconds = ttlSecondsOf(qualifiedName, 'expiration', byNumber ? opts : opts.expiration); + const evictionSeconds = byNumber ? undefined : ttlSecondsOf(qualifiedName, 'eviction', opts.eviction); + const scanIntervalSeconds = byNumber ? undefined : ttlSecondsOf(qualifiedName, 'scanInterval', opts.scanInterval); + const validatedReplicaExpirationMs = byNumber ? undefined : replicaExpirationMsOf(opts.replicaExpiration); + const validatedRefreshFraction = byNumber ? undefined : replicaRefreshFractionOf(opts.replicaRefreshFraction); const previousReplicaExpirationMs = replicaExpirationMs; const ownedCleanup = ownsCleanup(); - if (typeof opts === 'object' && opts.replicaExpiration != null) { - const validated = replicaExpirationMsOf(opts.replicaExpiration); - replicaExpirationMs = isRocksDB ? validated : 0; - } - if (typeof opts === 'object' && opts.replicaRefreshFraction != null) - replicaRefreshFraction = replicaRefreshFractionOf(opts.replicaRefreshFraction); - const declaredHere = typeof opts === 'object' && opts.fromSchema; + if (validatedReplicaExpirationMs !== undefined) + replicaExpirationMs = isRocksDB ? validatedReplicaExpirationMs : 0; + if (validatedRefreshFraction !== undefined) replicaRefreshFraction = validatedRefreshFraction; + const declaredHere = !byNumber && opts.fromSchema; const isolatedApplicationOwner = declaredHere && opts.isolatedApplicationOwner; + // a schema without a TTL does not clear one this thread's application code configured + const keepApplicationTtl = + declaredHere && applicationTtl !== 'none' && !expirationSeconds && !evictionSeconds && !scanIntervalSeconds; const preserveLoadedConfiguration = - typeof opts === 'object' && + !byNumber && (declaredHere || opts.replicaExpiration !== undefined || opts.replicaRefreshFraction !== undefined) && - opts.expiration === undefined && - opts.eviction === undefined && - opts.scanInterval === undefined; - if (((!ttlFromLoad && !declaredHere) || isolatedApplicationOwner) && !ttlConfiguredByApplication) { - ttlConfiguredByApplication = true; + ((opts.expiration === undefined && opts.eviction === undefined && opts.scanInterval === undefined) || + (keepApplicationTtl && applicationTtl === 'all')); + if (((!ttlFromLoad && !declaredHere) || isolatedApplicationOwner) && !ttlOwnedByThread) { + ttlOwnedByThread = true; // the scan owner may have changed with this: re-evaluate even if the interval did not lastCleanupInterval = undefined; } - if (typeof opts === 'number') { - expirationMs = opts * 1000; + if (byNumber) { + expirationMs = expirationSeconds * 1000; } else if (!preserveLoadedConfiguration) { - // `??` so an explicit 0 is treated as the user's chosen value, not as "missing" - expirationMs = (opts.expiration ?? 0) * 1000; - evictionMs = (opts.eviction ?? 0) * 1000; - recordScanInterval = (opts.scanInterval ?? 0) * 1000; + // kept here means the application set only the expiration: the rest is the schema's to clear + if (!keepApplicationTtl) expirationMs = (expirationSeconds ?? 0) * 1000; + evictionMs = (evictionSeconds ?? 0) * 1000; + recordScanInterval = (scanIntervalSeconds ?? 0) * 1000; + } + if (!preserveLoadedConfiguration && !ttlFromLoad) { + if (keepApplicationTtl || (!declaredHere && !byNumber)) applicationTtl = 'all'; + else if (declaredHere) applicationTtl = 'none'; + else if (applicationTtl === 'none') applicationTtl = 'expiration'; } - if (expirationMs < 0) throw new Error('Expiration can not be negative'); if (declaredHere && !preserveLoadedConfiguration) ttlDeclaredBySchema = true; + const scanWasScheduled = !preserveLoadedConfiguration && (expirationScanScheduled || cleanupTimer); if (!preserveLoadedConfiguration) { - // default to one quarter of the total expiration+eviction window - recordCleanupInterval = recordScanInterval || (expirationMs + evictionMs) / 4; - cleanupInterval = recordCleanupInterval; - expirationScanScheduled = true; + expirationScanScheduled = Boolean(recordScanInterval || expirationMs || evictionMs); + cleanupInterval = recordScanInterval || (expirationMs + evictionMs) / 4 || DEFAULT_CLEANUP_INTERVAL; + if (scanWasScheduled && !expirationScanScheduled) retainExpiryScan = true; } if ( replicaTableKnown && @@ -3978,7 +4031,7 @@ export function makeTable(options): TableResourceClass { } else if (replicaEvictionArmed) armReplicaEviction(); // Re-evaluate an existing table-level scan after an ownership-only declaration, but do not // create the default daily cleanup timer for a table that has only an @expiresAt field. - if (!preserveLoadedConfiguration || expirationScanScheduled || evictionMs) scheduleCleanup(); + if (expirationScanScheduled || evictionMs || retainExpiryScan) scheduleCleanup(); // @expiresAt has its own interval rather than the cleanup timer above. Arm it whenever a live // declaration introduces the attribute, including after this application already claimed TTL. if (expiresAtProperty && !recordExpirationInterval) runRecordExpirationEviction(); @@ -7709,7 +7762,14 @@ export function makeTable(options): TableResourceClass { // Evaluate at most once per table (on the first expiring write); later writes short-circuit on one check. if (!expirationWarningChecked) { expirationWarningChecked = true; - if (!expirationMs && !evictionMs && !expirationScanScheduled && !expiresAtProperty && !hasSourceGet) { + if ( + !expirationMs && + !evictionMs && + !expirationScanScheduled && + !retainExpiryScan && + !expiresAtProperty && + !hasSourceGet + ) { logger.warn?.( `A per-record expiresAt was set on table "${tableName}" which has no table-level expiration/eviction, no expiresAt attribute, and no source; expiration will not be reliably enforced (the eviction scan is only armed best-effort on write and does not survive a restart with no writes). Configure a table-level expiration/eviction or an indexed expiresAt attribute for reliable eviction.` ); @@ -12303,7 +12363,7 @@ export function makeTable(options): TableResourceClass { if (disposed || !maintenanceClosed) return; maintenanceClosed = false; lastCleanupInterval = undefined; - if (expirationScanScheduled || evictionMs) scheduleCleanup(); + if (expirationScanScheduled || evictionMs || retainExpiryScan) scheduleCleanup(); if (replicaEvictionArmed) scheduleReplicaSweep(); if (expiresAtProperty && !recordExpirationInterval) runRecordExpirationEviction(); } @@ -12390,14 +12450,15 @@ export function makeTable(options): TableResourceClass { defaultAllowReads.add(TableResource.prototype.allowRead); try { TableResource.updatedAttributes(); // on creation, update accessors as well - if (expirationMs || evictionMs) { + // anything but absent or zero is validated here: a peer's definition reaches a create unvalidated + if ((expirationMs !== undefined && expirationMs !== 0) || evictionMs !== 0) { ttlFromLoad = true; try { - TableResource.setTTLExpiration(expirationMs / 1000); + TableResource.setTTLExpiration({ expiration: (expirationMs ?? 0) / 1000, eviction: evictionMs / 1000 }); } finally { ttlFromLoad = false; } - } + } else if (retainExpiryScan) scheduleCleanup(); if (expiresAtProperty && !recordExpirationInterval) runRecordExpirationEviction(); } catch (error) { TableResource.cleanup(); @@ -14424,11 +14485,6 @@ export function makeTable(options): TableResourceClass { if (ownsCleanup()) { // run on the last thread so we aren't overloading lower-numbered threads if (cleanupTimer) clearTimeout(cleanupTimer); - if (!cleanupInterval) { - // no replacement pass is being scheduled, so nothing is left to settle a superseded one - settlePendingCleanup(); - return; - } // This pass adopts the awaiters of the pass whose timer it just cleared: they settle when // this pass's scan completes, so a reclamation run is never told the storage was reclaimed // before any scan ran. It has to run now, though — that run blocks its whole path on the @@ -14567,7 +14623,7 @@ export function makeTable(options): TableResourceClass { function runRecordExpirationEviction() { if (isReadOnlyMode()) return; // Periodically evict expired records, searching for records who expiresAt timestamp is before now - if (ownsStoreExpiration(primaryStore.path) || (ttlConfiguredByApplication && isDedicatedWorker())) { + if (ownsStoreExpiration(primaryStore.path) || (ttlOwnedByThread && isDedicatedWorker())) { // we want to run the pruning of expired records on only one thread so we don't have conflicts in evicting recordExpirationInterval = setInterval(() => { // go through each database and table and then search for expired entries @@ -15185,7 +15241,7 @@ export function makeTable(options): TableResourceClass { )); } function ownsCleanup(): boolean { - return ownsStoreMaintenance(primaryStore.path) || (ttlConfiguredByApplication && isDedicatedWorker()); + return ownsStoreMaintenance(primaryStore.path) || (ttlOwnedByThread && isDedicatedWorker()); } // an already-armed thread may own the timer only now (the main thread under threads: 0 loads // tables before it is worker 0), so the flag and the timer are settled on every wake diff --git a/resources/databases.ts b/resources/databases.ts index 466462801c..fdc303b0c3 100644 --- a/resources/databases.ts +++ b/resources/databases.ts @@ -44,6 +44,9 @@ import { makeTable, replicaExpirationMsOf, replicaRefreshFractionOf, + describeTtlValue, + savedTtlSeconds, + ttlSecondsOf, ignoreAlreadyDropped, acquireUpdateAttributesLock, releaseUpdateAttributesLock, @@ -1137,6 +1140,30 @@ export function catalogCreatedTime(table: { dbisDB: any; tableName: string; prim const createdTime = primaryCatalogRowFor(table)?.value?.createdTime; return Number.isFinite(createdTime) ? createdTime : undefined; } +function savedTtlOf(row: any): { expiration?: number; eviction?: number } { + return { expiration: savedTtlSeconds(row.expiration), eviction: savedTtlSeconds(row.eviction) }; +} +/** Whether a declared TTL field (undefined = not declared, 0 = cleared) differs from the catalog's. */ +function ttlChanged(declared: number | undefined, saved: unknown): boolean { + if (declared === undefined) return false; + return declared ? Number(saved) !== declared : saved !== undefined; +} +const warnedInvalidSavedTtl = new Set(); +/** + * A saved TTL field as a load applies it: a value no declaration could save now (a negative eviction or a + * non-numeric expiration an older create persisted) loads as absent, and the next declaration rewrites it. + */ +function loadedTtlSeconds(databaseName: string, tableName: string, field: string, saved: unknown) { + const seconds = savedTtlSeconds(saved); + const key = `${databaseName}.${tableName}.${field}`; + if (seconds === undefined && saved !== undefined && saved !== 0 && !warnedInvalidSavedTtl.has(key)) { + logger.warn( + `Ignoring the saved ${field} ${describeTtlValue(saved)} of ${databaseName}.${tableName}: not a valid TTL` + ); + warnedInvalidSavedTtl.add(key); + } + return seconds; +} /** * The table-level expiration and eviction (seconds) on disk, each present only when set. A class on a thread * that application code configured can hold a TTL the catalog does not. @@ -1148,14 +1175,25 @@ export function catalogExpiration(table: { dbisDB: any; tableName: string; prima // the row declareTable writes the settings to and initStores loads them from: a named primary row wins const named = table.primaryKey ? table.dbisDB?.getSync(`${table.tableName}/${table.primaryKey}`) : undefined; const row = named?.isPrimaryKey ? named : table.dbisDB?.getSync(`${table.tableName}/`); - // a create persists the declared value as given, which can be a numeric string - const expiration = Number(row?.expiration); - const eviction = Number(row?.eviction); + // a create persisted the declared value as given, which can be a numeric string + const expiration = savedTtlSeconds(row?.expiration); + const eviction = savedTtlSeconds(row?.eviction); const settings: { expiration?: number; eviction?: number } = {}; - if (expiration > 0 && Number.isFinite(expiration)) settings.expiration = expiration; - if (eviction > 0 && Number.isFinite(eviction)) settings.eviction = eviction; + if (expiration) settings.expiration = expiration; + if (eviction) settings.eviction = eviction; return settings; } +/** + * Arms the cleanup scan of every loaded table that has a TTL. Under `threads: 0` the main thread loads every + * table before it becomes worker 0, when no thread can own a scan yet. + */ +export function armTableCleanupScans(): void { + const loaded = getDatabases(); + // system is not enumerable + for (const database of new Set([loaded.system, ...Object.values(loaded)])) { + for (const Table of Object.values(database ?? {})) (Table as any)?._armCleanupScan?.(); + } +} /** * Backfills the stamp of a generation created on a build that stored none, once a peer's stamped definition * proves which generation it is. Only an absent stamp is written; false when one exists or the table is gone. @@ -2158,8 +2196,8 @@ function initStores( const audit = typeof primaryAttribute.audit === 'boolean' ? primaryAttribute.audit : envGet(CONFIG_PARAMS.LOGGING_AUDITLOG); const trackDeletes = primaryAttribute.trackDeletes; - const expiration = primaryAttribute.expiration; - const eviction = primaryAttribute.eviction; + const expiration = loadedTtlSeconds(databaseName, tableName, 'expiration', primaryAttribute.expiration); + const eviction = loadedTtlSeconds(databaseName, tableName, 'eviction', primaryAttribute.eviction); const replicaExpiration = primaryAttribute.replicaExpiration; const replicaRefreshFraction = primaryAttribute.replicaRefreshFraction; const sealed = primaryAttribute.sealed; @@ -2338,7 +2376,7 @@ function initStores( if (!destination) requestComputedIndexRepair(table); if (replicaExpiration !== undefined || replicaRefreshFraction !== undefined) table.setTTLExpiration({ replicaExpiration, replicaRefreshFraction, fromSchema: true }); - table._adoptCatalogExpiration(expiration, eviction); + table._adoptCatalogExpiration(expiration, eviction, primaryAttribute.expiryScan === true); } else { if (recreateForTableIdChange) table.cleanup(); table = setTable( @@ -2358,6 +2396,7 @@ function initStores( replicaExpirationMS: replicaExpirationMsOf(replicaExpiration), replicaRefreshFraction: replicaRefreshFractionOf(replicaRefreshFraction), replicaDeadlines: Boolean(tableDef.replicaDeadlines), + expiryScan: primaryAttribute.expiryScan === true, cacheControl, trackDeletes, tableName, @@ -2905,9 +2944,10 @@ interface TableDefinition { path?: string; /** Kept when a peer's definition creates the table here, so every copy of a generation shares one stamp. */ createdTime?: number; - expiration?: number; - eviction?: number; - scanInterval?: number; + /** Seconds; 0 or null clears a saved value, undefined (a caller not authoring the schema) keeps it. */ + expiration?: number | null; + eviction?: number | null; + scanInterval?: number | null; /** RocksDB-only replica lifetime in seconds; 0 disables admission. LMDB always proxies. */ replicaExpiration?: number; /** RocksDB-only renewal fraction in (0, 1]; 1 never renews. */ @@ -4360,18 +4400,27 @@ function declareTable(target: TableTarget, tableDefinition: T if (isBranchIdentity(databaseName)) { throw new ClientError(`'${databaseName}' is in use as a branch store identity and cannot be a database name`); } + // Before anything is opened, created or persisted. A peer's values reach only a table this call creates, as + // a catalog load does (see the end of this function). + if (origin !== 'cluster') { + const qualifiedName = `${databaseName}.${tableName}`; + expiration = ttlSecondsOf(qualifiedName, 'expiration', expiration); + eviction = ttlSecondsOf(qualifiedName, 'eviction', eviction); + scanInterval = ttlSecondsOf(qualifiedName, 'scanInterval', scanInterval); + } const rootStore = target.rootStore(databaseName, tableName); const tables = target.tables(databaseName); logger.trace(`Defining ${tableName} in ${databaseName}`); let Table = tables?.[tableName]; + const namesOneTtlField = origin !== 'cluster' && (expiration === undefined) !== (eviction === undefined); + // the saved value of the field such a declaration omits, from the row persistTableSettings writes + let inheritedTtl: { expiration?: number; eviction?: number } | undefined; if (rootStore.status === 'closed') { throw new Error(`Can not use a closed data store for ${tableName}`); } - // A re-declaration persists these to the catalog, where a negative expiration fails every later load, and - // nothing it changes before then (index builds included) is rolled back. + // A re-declaration persists these to the catalog, and nothing it changes before then (index builds included) + // is rolled back. if (Table && origin !== 'cluster') { - if (+expiration < 0 || +eviction < 0) - throw new ClientError(`Expiration and eviction of '${databaseName}.${tableName}' can not be negative`); if (replicaExpiration != null) replicaExpirationMsOf(replicaExpiration); if (replicaRefreshFraction != null) replicaRefreshFractionOf(replicaRefreshFraction); } @@ -5786,10 +5835,15 @@ function declareTable(target: TableTarget, tableDefinition: T // A peer's expiration reaches a table this call created through makeTable, as a catalog load. It never // configures an existing table, and applying it here would mark the TTL as this thread's own declaration. if (origin === 'cluster') expiration = eviction = scanInterval = undefined; + if (inheritedTtl) { + expiration ??= inheritedTtl.expiration ?? 0; + eviction ??= inheritedTtl.eviction ?? 0; + } if ( expiration || eviction || scanInterval || + ((expiration === 0 || eviction === 0) && (Table.expirationMS || Table.evictionMS)) || replicaExpiration !== undefined || replicaRefreshFraction !== undefined || attributes.some((attribute) => attribute.expiresAt) @@ -5819,6 +5873,7 @@ function declareTable(target: TableTarget, tableDefinition: T } function persistTableSettings(attribute: any, dbiKey: string) { const attributeDescriptor = attributesDbi.getSync(dbiKey) || {}; + if (namesOneTtlField) inheritedTtl = savedTtlOf(attributeDescriptor); // Persist schemaDefined when the explicit live value disagrees with disk. Without this, // a stale `false` (from a v4-era write or replicated event) survives every reload: the // in-memory re-assert in the existing-Table branch only fixes the worker that ran @table, @@ -5833,8 +5888,8 @@ function declareTable(target: TableTarget, tableDefinition: T (typeof audit === 'boolean' && audit !== attributeDescriptor.audit) || (sealed !== undefined && sealed !== attributeDescriptor.sealed) || (replicate !== undefined && replicate !== attributeDescriptor.replicate) || - (expiration && +expiration !== +attributeDescriptor.expiration) || - (eviction && +eviction !== +attributeDescriptor.eviction) || + ttlChanged(expiration, attributeDescriptor.expiration) || + ttlChanged(eviction, attributeDescriptor.eviction) || (replicaExpiration != null && replicaExpiration !== attributeDescriptor.replicaExpiration) || (replicaRefreshFraction != null && replicaRefreshFraction !== attributeDescriptor.replicaRefreshFraction) || attribute.type !== attributeDescriptor.type) @@ -5842,13 +5897,23 @@ function declareTable(target: TableTarget, tableDefinition: T exclusiveLock(); const currentPrimaryAttribute = attributesDbi.getSync(dbiKey); if (!currentPrimaryAttribute || tableIsDropping(currentPrimaryAttribute, dbiKey)) return; + if (namesOneTtlField) inheritedTtl = savedTtlOf(currentPrimaryAttribute); const updatedPrimaryAttribute = { ...currentPrimaryAttribute }; if (typeof audit === 'boolean') { if (audit) Table.enableAuditing(); updatedPrimaryAttribute.audit = audit; } - if (expiration) updatedPrimaryAttribute.expiration = +expiration; - if (eviction) updatedPrimaryAttribute.eviction = +eviction; + // Rows already stamped under a TTL being cleared keep their expiry, and only a cleanup scan removes + // them: a load of a table marked this way keeps the daily scan a table without a TTL would not arm. + if ( + (expiration === 0 && savedTtlSeconds(currentPrimaryAttribute.expiration)) || + (eviction === 0 && savedTtlSeconds(currentPrimaryAttribute.eviction)) + ) + updatedPrimaryAttribute.expiryScan = true; + if (expiration) updatedPrimaryAttribute.expiration = expiration; + else if (expiration === 0) delete updatedPrimaryAttribute.expiration; + if (eviction) updatedPrimaryAttribute.eviction = eviction; + else if (eviction === 0) delete updatedPrimaryAttribute.eviction; if (replicaExpiration != null) { replicaExpirationMsOf(replicaExpiration); updatedPrimaryAttribute.replicaExpiration = replicaExpiration; diff --git a/resources/defineTable.ts b/resources/defineTable.ts index 2b882fa40c..726b577000 100644 --- a/resources/defineTable.ts +++ b/resources/defineTable.ts @@ -401,6 +401,8 @@ function compileTypeDef(name: string, shape: Shape, options: DefineTableOptions) properties, schemaDefined: true, ...tableOptions, + expiration: tableOptions.expiration ?? null, + eviction: tableOptions.eviction ?? null, }; } diff --git a/resources/graphql.ts b/resources/graphql.ts index c077d48ff2..3cdc66d914 100644 --- a/resources/graphql.ts +++ b/resources/graphql.ts @@ -288,7 +288,9 @@ async function processGraphQLSchema( // Boolean directive args arrive as actual booleans; tolerate string forms too. if (typeDef.randomAccessFields !== undefined) typeDef.randomAccessFields = typeDef.randomAccessFields === true || typeDef.randomAccessFields === 'true'; - // schema is authoritative for cacheControl: null (vs undefined) clears a removed directive on reload; + typeDef.expiration ??= null; + typeDef.eviction ??= null; + // schema is authoritative for the TTL and cacheControl: null (vs undefined) clears a removed directive on reload; // coerce to a header-safe string so a non-string arg or CRLF can't reach the HTTP response typeDef.cacheControl = typeDef.cacheControl == null diff --git a/server/threads/socketRouter.ts b/server/threads/socketRouter.ts index 989b7e801e..ef4f399a65 100644 --- a/server/threads/socketRouter.ts +++ b/server/threads/socketRouter.ts @@ -21,6 +21,7 @@ import { startTransactionLogCooling } from '../transactionLogCooling.ts'; import { applyConfiguredV8Flags } from './v8Flags.ts'; import { startLongLivedTransactionReporting } from '../../resources/longLivedTransactions.ts'; import { initializeFullTextQueryPolicy } from '../../resources/indexes/fullTextNativeBinding.ts'; +import { armTableCleanupScans } from '../../resources/databases.ts'; import { effectiveHttpThreadCount, isWorkerPoolActive, @@ -108,6 +109,10 @@ export async function startHTTPThreads(threadCount = 2, dynamicThreads?: boolean if (isProcessShuttingDown()) return []; await threadServer.startServers(); if (isProcessShuttingDown()) return []; + // After the components' declarations, so a stale saved TTL never arms a scan that a schema replaces. + // Workers need no such pass: they have a worker index before loading components, so each table + // arms its own scan as it loads. + armTableCleanupScans(); // startServers() schedules listener startup after loading components; await its cached // batch so a bind failure reaches bin/run.ts and exits non-zero in single-thread mode too. await threadServer.listenOnPorts(); diff --git a/unitTests/resources/createTableCatalogOrder.test.js b/unitTests/resources/createTableCatalogOrder.test.js index f686292927..594fc32aaf 100644 --- a/unitTests/resources/createTableCatalogOrder.test.js +++ b/unitTests/resources/createTableCatalogOrder.test.js @@ -191,7 +191,11 @@ describe('create table catalog write order', () => { 'a create that fails before makeTable must not register the class' ); assert.strictEqual(releasedReclamationHandlers.length, 0, 'nothing was registered before makeTable'); - assert.throws(() => table({ ...definition(), expiration: -1 }), /Expiration can not be negative/); + // a peer's definition is not validated at the declaration boundary, so its bad value fails in makeTable + assert.throws( + () => table({ ...definition(), origin: 'cluster', expiration: -1 }), + /must be a non-negative number of seconds/ + ); assert.strictEqual( databases.test[tableName], undefined, diff --git a/unitTests/resources/tableExpirationPersistence.test.js b/unitTests/resources/tableExpirationPersistence.test.js index 2c6916d541..335882aed9 100644 --- a/unitTests/resources/tableExpirationPersistence.test.js +++ b/unitTests/resources/tableExpirationPersistence.test.js @@ -10,8 +10,10 @@ const { resetDatabases, table, } = require('#src/resources/databases'); +const { defineTable, types } = require('#src/resources/defineTable'); +const { loadGQLSchema } = require('#src/resources/graphql'); -const DATABASE = 'test'; +const DATABASE = 'ttl_persistence'; async function commitCatalog(Table) { if (Table.dbisDB.committed) await Table.dbisDB.committed; @@ -222,6 +224,22 @@ describe('table-level expiration persistence (harper-pro#1179)', () => { assert.strictEqual(Created.expirationMS, 3600000); }); + it("rejects a peer definition's invalid TTL on a table it creates, naming the field", function () { + for (const [suffix, settings, field] of [ + ['Eviction', { expiration: 60, eviction: -1 }, 'eviction'], + ['EvictionOnly', { eviction: -1 }, 'eviction'], + ['NonNumeric', { expiration: 'soon' }, 'expiration'], + ]) { + const tableName = `ExpiryPeerInvalid${suffix}`; + assert.throws( + () => keyed(tableName, { origin: 'cluster', schemaDefined: undefined, ...settings }), + new RegExp(`^Error: ${field} of '${DATABASE}.${tableName}' must be a non-negative number`), + suffix + ); + assert.strictEqual(databases[DATABASE][tableName], undefined, suffix); + } + }); + it("leaves an existing table's settings alone when a peer definition differs", async function () { const tableName = 'ExpiryPeerExisting'; const Local = keyed(tableName, { expiration: 600 }); @@ -245,4 +263,246 @@ describe('table-level expiration persistence (harper-pro#1179)', () => { assert.strictEqual(Local.evictionMS, 0); assert.deepStrictEqual(catalogExpiration(Local), { expiration: 600 }); }); + it('clears a saved expiration and eviction that a re-declaration sets to 0, null or "0" (#3934)', async function () { + for (const [suffix, cleared] of [ + ['Zero', { expiration: 0, eviction: 0 }], + ['Null', { expiration: null, eviction: null }], + ['String', { expiration: '0', eviction: '0' }], + ]) { + const tableName = `ExpiryCleared${suffix}`; + const Declared = keyed(tableName, { expiration: 600, eviction: 300 }); + keyed(tableName, cleared); + await commitCatalog(Declared); + const { value } = primaryRow(Declared); + assert(!('expiration' in value) && !('eviction' in value), `${suffix}: ${JSON.stringify(value)}`); + assert.deepStrictEqual(catalogExpiration(Declared), {}, suffix); + assert(!Declared.expirationMS, `${suffix}: this thread still applies ${Declared.expirationMS}ms`); + assert.strictEqual(Declared.evictionMS, 0, suffix); + + delete databases[DATABASE][tableName]; + resetDatabases(); + const Restarted = getDatabases()[DATABASE][tableName]; + assert(!Restarted.expirationMS, `${suffix}: a restart loads ${Restarted.expirationMS}ms`); + assert.strictEqual(Restarted.evictionMS, 0, suffix); + } + }); + + it('clears only the setting a re-declaration sets to 0', async function () { + const tableName = 'ExpiryClearedEvictionOnly'; + const Declared = keyed(tableName, { expiration: 600, eviction: 300 }); + keyed(tableName, { expiration: 600, eviction: 0 }); + await commitCatalog(Declared); + assert.deepStrictEqual(catalogExpiration(Declared), { expiration: 600 }); + assert.strictEqual(Declared.evictionMS, 0); + }); + + it('drops a TTL this thread loaded from the catalog when its re-declaration clears it', async function () { + const tableName = 'ExpiryLoadedThenCleared'; + const Loaded = keyed(tableName); + await writeCatalogSettings(Loaded, { expiration: 120, eviction: 60 }); + resetDatabases(); + assert.strictEqual(Loaded.expirationMS, 120000); + + keyed(tableName, { expiration: 0 }); + await commitCatalog(Loaded); + assert(!Loaded.expirationMS, `this thread still applies ${Loaded.expirationMS}ms`); + assert.deepStrictEqual(catalogExpiration(Loaded), { eviction: 60 }, 'an omitted eviction stays saved'); + assert.strictEqual(Loaded.evictionMS, 60000, 'and stays applied'); + }); + + it('keeps the saved value of the setting a re-declaration omits, on disk and in memory', async function () { + const tableName = 'ExpiryPartialUpdate'; + const Declared = keyed(tableName, { expiration: 60, eviction: 3600 }); + keyed(tableName, { expiration: 120 }); + await commitCatalog(Declared); + assert.deepStrictEqual(catalogExpiration(Declared), { expiration: 120, eviction: 3600 }); + assert.strictEqual(Declared.expirationMS, 120000); + assert.strictEqual(Declared.evictionMS, 3600000); + }); + + it('keeps the omitted setting as the row it writes holds it, not as an earlier read saw it', async function () { + const tableName = 'ExpiryInheritedUnderLock'; + const Declared = keyed(tableName, { expiration: 60, eviction: 300 }); + await commitCatalog(Declared); + const { key } = primaryRow(Declared); + const catalog = Declared.dbisDB; + const originalGetSync = catalog.getSync; + let raced = false; + catalog.getSync = function (readKey, ...rest) { + const value = originalGetSync.call(this, readKey, ...rest); + if (readKey === key && !raced) { + raced = true; + // another thread saves a new eviction right after this declaration's first read of the row + catalog.putSync(key, { ...value, eviction: 3600 }); + } + return value; + }; + try { + keyed(tableName, { expiration: 120 }); + } finally { + catalog.getSync = originalGetSync; + } + await commitCatalog(Declared); + assert.deepStrictEqual(catalogExpiration(Declared), { expiration: 120, eviction: 3600 }); + assert.strictEqual(Declared.evictionMS, 3600000); + }); + + it('does not clear a TTL application code configured when the schema declares none', async function () { + const tableName = 'ExpiryApplicationKept'; + const Configured = keyed(tableName); + Configured.setTTLExpiration({ expiration: 30, eviction: 10 }); + keyed(tableName, { expiration: null, eviction: null }); + assert.strictEqual(Configured.expirationMS, 30000); + assert.strictEqual(Configured.evictionMS, 10000); + }); + + it('keeps a TTL application code set through a re-declaration with none, isolated or not (QA-5424)', async function () { + for (const isolatedApplicationOwner of [true, false]) { + const tableName = `ExpiryApplicationRedeclared${isolatedApplicationOwner ? 'Isolated' : 'Shared'}`; + const Configured = keyed(tableName, { expiration: null, eviction: null, isolatedApplicationOwner }); + Configured.setTTLExpiration({ expiration: 30, eviction: 10 }); + keyed(tableName, { expiration: null, eviction: null, isolatedApplicationOwner }); + assert.strictEqual(Configured.expirationMS, 30000, `isolatedApplicationOwner: ${isolatedApplicationOwner}`); + assert.strictEqual(Configured.evictionMS, 10000, `isolatedApplicationOwner: ${isolatedApplicationOwner}`); + await commitCatalog(Configured); + assert.deepStrictEqual(catalogExpiration(Configured), {}, 'an application TTL is never saved'); + } + }); + + it('keeps only the expiration a number-form call set over a schema TTL the schema then drops', async function () { + const tableName = 'ExpiryNumberFormOverSchema'; + const Declared = keyed(tableName, { expiration: 60, eviction: 3600, scanInterval: 3600 }); + Declared.setTTLExpiration(30); + keyed(tableName, { expiration: null, eviction: null }); + assert.strictEqual(Declared.expirationMS, 30000); + assert.strictEqual(Declared.evictionMS, 0, "the eviction was the schema's"); + keyed(tableName, { expiration: null, eviction: null }); + assert.strictEqual(Declared.expirationMS, 30000, 'a further declaration without a TTL keeps it'); + }); + + it("clears a TTL an isolated application's schema declared when it declares none", async function () { + const tableName = 'ExpiryIsolatedSchemaCleared'; + const Declared = keyed(tableName, { expiration: 600, eviction: 300, isolatedApplicationOwner: true }); + assert.strictEqual(Declared.expirationMS, 600000); + keyed(tableName, { expiration: null, eviction: null, isolatedApplicationOwner: true }); + await commitCatalog(Declared); + assert.deepStrictEqual(catalogExpiration(Declared), {}); + assert(!Declared.expirationMS, `this thread still applies ${Declared.expirationMS}ms`); + assert.strictEqual(Declared.evictionMS, 0); + }); + + it('clears a schema TTL that replaced the one application code set, when the schema drops it', async function () { + for (const isolatedApplicationOwner of [true, false]) { + const tableName = `ExpirySchemaReplacedApplication${isolatedApplicationOwner ? 'Isolated' : 'Shared'}`; + const Configured = keyed(tableName, { isolatedApplicationOwner }); + Configured.setTTLExpiration({ expiration: 30 }); + keyed(tableName, { expiration: 120, isolatedApplicationOwner }); + assert.strictEqual(Configured.expirationMS, 120000); + keyed(tableName, { expiration: null, eviction: null, isolatedApplicationOwner }); + assert( + !Configured.expirationMS, + `isolatedApplicationOwner ${isolatedApplicationOwner} kept ${Configured.expirationMS}ms` + ); + } + }); + + it('loads a saved value no declaration could save now as absent', async function () { + const tableName = 'ExpiryLegacyInvalid'; + const Legacy = keyed(tableName); + await writeCatalogSettings(Legacy, { expiration: { legacy: 1n }, eviction: -1 }); + delete databases[DATABASE][tableName]; + resetDatabases(); + const Restarted = getDatabases()[DATABASE][tableName]; + assert(Restarted, 'the table must still load'); + assert(!Restarted.expirationMS); + assert.strictEqual(Restarted.evictionMS, 0); + keyed(tableName, { expiration: 0, eviction: 0 }); + await commitCatalog(Restarted); + const { value } = primaryRow(Restarted); + assert(!('expiration' in value) && !('eviction' in value), 'the next declaration rewrites the row'); + }); + + it('brings a loaded class up to date when another thread clears the catalog (#3934)', async function () { + const tableName = 'ExpiryAdoptedClear'; + const Loaded = keyed(tableName); + await writeCatalogSettings(Loaded, { expiration: 120, eviction: 60 }); + resetDatabases(); + assert.strictEqual(Loaded.expirationMS, 120000); + + const { key, value } = primaryRow(Loaded); + const cleared = { ...value }; + delete cleared.expiration; + delete cleared.eviction; + Loaded.dbisDB.putSync(key, cleared); + await commitCatalog(Loaded); + resetDatabases(); + assert(!Loaded.expirationMS, `the reload kept ${Loaded.expirationMS}ms`); + assert.strictEqual(Loaded.evictionMS, 0); + }); + + it('keeps a saved TTL through callers that do not author it', async function () { + const tableName = 'ExpiryAddAttribute'; + const Declared = keyed(tableName, { expiration: 600, eviction: 300 }); + await Declared.addAttributes([{ name: 'added', type: 'String' }]); + await commitCatalog(Declared); + assert.deepStrictEqual(catalogExpiration(Declared), { expiration: 600, eviction: 300 }); + assert.strictEqual(Declared.expirationMS, 600000); + }); + + it('clears a TTL that a @table directive no longer declares (#3934)', async function () { + const tableName = 'ExpiryGraphqlRemoved'; + await loadGQLSchema(`type ${tableName} @table(database: "${DATABASE}", expiration: 600, eviction: 300) { + id: ID @primaryKey + }`); + const Declared = databases[DATABASE][tableName]; + await commitCatalog(Declared); + assert.deepStrictEqual(catalogExpiration(Declared), { expiration: 600, eviction: 300 }); + + await loadGQLSchema(`type ${tableName} @table(database: "${DATABASE}") { + id: ID @primaryKey + }`); + await commitCatalog(Declared); + assert.deepStrictEqual(catalogExpiration(Declared), {}); + assert(!Declared.expirationMS, `this thread still applies ${Declared.expirationMS}ms`); + assert.strictEqual(Declared.evictionMS, 0); + }); + + it('clears a TTL that defineTable options no longer declare (#3934)', async function () { + const tableName = 'ExpiryDefineTableRemoved'; + const shape = { id: types.id.primaryKey }; + const Declared = defineTable(tableName, shape, { database: DATABASE, expiration: 600 }); + await commitCatalog(Declared); + assert.deepStrictEqual(catalogExpiration(Declared), { expiration: 600 }); + + defineTable(tableName, shape, { database: DATABASE }); + await commitCatalog(Declared); + assert.deepStrictEqual(catalogExpiration(Declared), {}); + assert(!Declared.expirationMS, `this thread still applies ${Declared.expirationMS}ms`); + }); + + it('rejects an invalid TTL before a create, naming the table and field (#1286, #3935)', async function () { + for (const [field, value] of [ + ['expiration', -1], + ['eviction', -1], + ['scanInterval', -1], + ['expiration', 'soon'], + ['eviction', Infinity], + ['expiration', Number.NaN], + ['expiration', 1e308], + ['expiration', ''], + ['expiration', true], + ['eviction', [5]], + ]) { + const tableName = `ExpiryInvalidCreate_${field}_${String(value).replace(/\W/g, '')}`; + assert.throws( + () => keyed(tableName, { [field]: value }), + (error) => + error.statusCode === 400 && + error.message.includes(field) && + error.message.includes(`${DATABASE}.${tableName}`), + `${field}: ${value}` + ); + assert.strictEqual(databases[DATABASE][tableName], undefined, `${field}: ${value} created the table`); + } + }); }); diff --git a/unitTests/resources/ttlCleanupInterval.test.js b/unitTests/resources/ttlCleanupInterval.test.js index f224328d86..5ccc99bf62 100644 --- a/unitTests/resources/ttlCleanupInterval.test.js +++ b/unitTests/resources/ttlCleanupInterval.test.js @@ -1,12 +1,46 @@ require('../testUtils'); const assert = require('assert'); const { setupTestDBPath } = require('../testUtils'); -const { table } = require('#src/resources/databases'); +const { + table, + databases, + getDatabases, + resetDatabases, + armTableCleanupScans, + scopedTableFactory, +} = require('#src/resources/databases'); const { makeTable } = require('#src/resources/Table'); const { setMainIsWorker } = require('#src/server/threads/manageThreads'); const { waitFor } = require('../waitFor'); const { setTimeout: sleep } = require('node:timers/promises'); +async function timersArmedBy(run, settleMs = 0) { + const originalSetTimeout = global.setTimeout; + const delays = []; + global.setTimeout = (callback, delay, ...args) => { + delays.push(delay); + return originalSetTimeout(callback, delay, ...args); + }; + try { + run(); + if (settleMs) await sleep(settleMs); + } finally { + global.setTimeout = originalSetTimeout; + } + return delays; +} +// the TTLs these tests clear scan every 15 s or less +const isDailyCadence = (delay) => (delay > 15_000 && delay <= 86_400_000) || isNextDailySlot(delay); +// a first daily scan with nothing pending lands on the next whole day since the start of the year +function isNextDailySlot(delay) { + const startOfYear = new Date(); + startOfYear.setMonth(0, 1); + startOfYear.setHours(0, 0, 0); // keeps the milliseconds, as scheduleCleanup does + const now = Date.now(); + const slot = Math.ceil((now - startOfYear.getTime()) / 86_400_000) * 86_400_000 + startOfYear.getTime(); + return Math.abs(delay - (slot - now)) < 5000; +} + // The cleanup scan is what removes an expired record: reads return it until then. Its interval has to come // from the table's TTL whichever way the TTL was set. A worker that learns of a table from the catalog (a // table created at runtime on another thread, or any table after a restart) arms it with the number form. @@ -100,6 +134,31 @@ describe('TTL cleanup scan interval', () => { } }); + it("an isolated application's schema without a TTL leaves the scan for one application code set (QA-5424)", async () => { + const definition = { + table: 'TtlIsolatedApplicationKept', + database: databaseName, + expiration: null, + eviction: null, + attributes: [{ name: 'id', isPrimaryKey: true }], + }; + // the table as a previous start left it; application code (jsResource) then loads before the schema + const Table = table({ table: definition.table, database: databaseName, attributes: definition.attributes }); + const isolatedTable = scopedTableFactory(undefined, true); + try { + Table.sourcedFrom({ get: (id) => ({ id }) }, { expiration: 0.5 }); + isolatedTable(definition); + isolatedTable(definition); + assert.strictEqual(Table.expirationMS, 500); + await Table.put(1, { id: 1 }); + await Table.primaryStore.committed; + assert(stored(Table, 1)); + await waitFor(() => !stored(Table, 1), { timeout: 5000, message: 'the expired record was never evicted' }); + } finally { + Table.cleanup(); + } + }); + it('the number form keeps a configured scanInterval', () => { const Table = table({ table: 'TtlScanInterval', @@ -154,4 +213,229 @@ describe('TTL cleanup scan interval', () => { Hydrated.cleanup(); } }); + it('an eviction-only TTL adopted from the catalog arms the scan (#3935)', async () => { + const Declared = table({ + table: 'TtlAdoptedEvictionOnly', + database: databaseName, + attributes: [{ name: 'id', isPrimaryKey: true }], + }); + const Hydrated = hydrate(Declared, 'TtlAdoptedEvictionOnly', {}); + try { + const delays = await timersArmedBy(() => Hydrated._adoptCatalogExpiration(0, 0.05)); + assert.strictEqual(Hydrated.expirationMS, 0); + assert.strictEqual(Hydrated.evictionMS, 50); + assert.strictEqual(delays.length, 1, 'adopting an eviction-only TTL must arm the scan'); + await Hydrated.put(1, { id: 1 }, { expiresAt: Date.now() }); + await Hydrated.primaryStore.committed; + await waitFor(() => !stored(Hydrated, 1), { timeout: 5000, message: 'the evictable record was never evicted' }); + } finally { + Hydrated.cleanup(); + } + }); + + it('adopting a cleared TTL leaves the scan at the daily default (#3934)', async () => { + const Declared = table({ + table: 'TtlAdoptedClear', + database: databaseName, + attributes: [{ name: 'id', isPrimaryKey: true }], + }); + const Hydrated = hydrate(Declared, 'TtlAdoptedClear', { expirationMS: 60_000 }); + try { + const delays = await timersArmedBy(() => Hydrated._adoptCatalogExpiration(undefined, undefined), 200); + assert.strictEqual(Hydrated.expirationMS, 0); + assert.strictEqual(Hydrated.evictionMS, 0); + assert(delays.some(isDailyCadence), `no daily scan armed: ${delays}`); + } finally { + Hydrated.cleanup(); + } + }); + + it('clearing a TTL with zeros keeps a scan for per-record expiry instead of disabling it', async () => { + const Table = table({ + table: 'TtlZeroed', + database: databaseName, + attributes: [{ name: 'id', isPrimaryKey: true }], + }); + try { + Table.setTTLExpiration({ expiration: 60 }); + const delays = await timersArmedBy(() => Table.setTTLExpiration({ expiration: 0, eviction: 0 }), 200); + assert(delays.some(isDailyCadence), `the scan was disabled instead of moved to the daily default: ${delays}`); + } finally { + Table.cleanup(); + } + }); + + it('rejects a negative or non-finite value without applying any of it (#3935)', () => { + const Table = table({ + table: 'TtlRejected', + database: databaseName, + attributes: [{ name: 'id', isPrimaryKey: true }], + }); + try { + Table.setTTLExpiration({ expiration: 10, eviction: 5 }); + for (const opts of [ + { expiration: 1, eviction: -1 }, + { expiration: -1, eviction: 1 }, + { expiration: 1, scanInterval: -1 }, + { expiration: Infinity }, + { expiration: Number.NaN }, + -1, + ]) { + assert.throws( + () => Table.setTTLExpiration(opts), + /must be a non-negative number of seconds/, + JSON.stringify(opts) + ); + assert.strictEqual(Table.expirationMS, 10_000, `${JSON.stringify(opts)} changed the expiration`); + assert.strictEqual(Table.evictionMS, 5_000, `${JSON.stringify(opts)} changed the eviction`); + } + } finally { + Table.cleanup(); + } + }); + + it('a table loaded before the main thread became a worker arms its scan once it is one (#2831)', async () => { + const tableName = 'TtlLoadedBeforePromotion'; + const definition = { table: tableName, database: databaseName, attributes: [{ name: 'id', isPrimaryKey: true }] }; + const Declared = table(definition); + // a restart: the row outlives this class, whose scan stops with it; the TTL comes after the commit, so + // no scan of this class can reach the row + const expiresAt = Date.now() + 1000; + await Declared.put(1, { id: 1 }, { expiresAt }); + await Declared.primaryStore.committed; + table({ ...definition, expiration: 0.05 }); + Declared.cleanup(); + delete databases[databaseName][tableName]; + let Loaded; + try { + // threads: 0 loads every table on the main thread before setMainIsWorker(true) + setMainIsWorker(false); + resetDatabases(); + Loaded = getDatabases()[databaseName][tableName]; + } finally { + setMainIsWorker(true); + } + try { + assert.strictEqual(Loaded.expirationMS, 50); + await sleep(Math.max(0, expiresAt + 300 - Date.now())); + assert(stored(Loaded, 1), 'a scan ran although no thread owned it at load'); + armTableCleanupScans(); + await waitFor(() => !stored(Loaded, 1), { timeout: 5000, message: 'the expired record was never evicted' }); + } finally { + Loaded.cleanup(); + } + }); + + it('a TTL cleared once the main thread is a worker still scans the rows it stamped (#2831, #3934)', async () => { + const Declared = table({ + table: 'TtlClearedAfterPromotion', + database: databaseName, + attributes: [{ name: 'id', isPrimaryKey: true }], + }); + let Hydrated; + try { + // threads: 0 loads the table before the main thread owns any scan, then the schema clears its TTL + setMainIsWorker(false); + Hydrated = hydrate(Declared, 'TtlClearedAfterPromotion', { expirationMS: 60_000 }); + } finally { + setMainIsWorker(true); + } + try { + const delays = await timersArmedBy( + () => Hydrated.setTTLExpiration({ expiration: 0, eviction: 0, fromSchema: true }), + 200 + ); + assert(delays.some(isDailyCadence), `no scan armed for the rows the cleared TTL stamped: ${delays}`); + } finally { + Hydrated.cleanup(); + } + }); + + it('keeps a scan for rows a cleared TTL stamped after a restart (#3934)', async () => { + const tableName = 'TtlClearedThenRestarted'; + const attributes = () => [{ name: 'id', isPrimaryKey: true }]; + const Declared = table({ table: tableName, database: databaseName, expiration: 60, attributes: attributes() }); + table({ table: tableName, database: databaseName, expiration: 0, attributes: attributes() }); + if (Declared.dbisDB.committed) await Declared.dbisDB.committed; + const primaryRow = Declared.dbisDB.getSync(`${tableName}/`) ?? Declared.dbisDB.getSync(`${tableName}/id`); + assert.strictEqual(primaryRow.expiryScan, true, JSON.stringify(primaryRow)); + Declared.cleanup(); + delete databases[databaseName][tableName]; + let Restarted; + try { + // loaded with no scan owner, so the only timer below is the one this table arms + setMainIsWorker(false); + resetDatabases(); + Restarted = getDatabases()[databaseName][tableName]; + } finally { + setMainIsWorker(true); + } + try { + assert(!Restarted.expirationMS); + const delays = await timersArmedBy(() => Restarted._armCleanupScan()); + assert(delays.length === 1 && isNextDailySlot(delays[0]), `no daily scan armed: ${delays}`); + } finally { + Restarted.cleanup(); + } + }); + + it('a loaded class arms the scan when a reload finds the catalog marked expiryScan', async () => { + const Declared = table({ + table: 'TtlReloadExpiryScan', + database: databaseName, + attributes: [{ name: 'id', isPrimaryKey: true }], + }); + const Hydrated = hydrate(Declared, 'TtlReloadExpiryScan', {}); + try { + const delays = await timersArmedBy(() => Hydrated._adoptCatalogExpiration(undefined, undefined, true)); + assert(delays.length === 1 && isNextDailySlot(delays[0]), `no daily scan armed: ${delays}`); + } finally { + Hydrated.cleanup(); + } + }); + + it('keeps a scan for rows a cleared TTL stamped when maintenance resumes (#3934)', async () => { + const Table = table({ + table: 'TtlClearedThenResumed', + database: databaseName, + attributes: [{ name: 'id', isPrimaryKey: true }], + }); + try { + Table.setTTLExpiration({ expiration: 60 }); + Table.setTTLExpiration({ expiration: 0, eviction: 0 }); + await Table.closeMaintenance(); + const delays = await timersArmedBy(() => Table.resumeMaintenance()); + assert(delays.some(isDailyCadence), `resuming maintenance armed no scan: ${delays}`); + } finally { + Table.cleanup(); + } + }); + + it('arms no scan for a table with no TTL', async () => { + const Table = table({ + table: 'TtlNoneToArm', + database: databaseName, + attributes: [{ name: 'id', isPrimaryKey: true }], + }); + try { + assert.deepStrictEqual(await timersArmedBy(() => Table._armCleanupScan()), []); + } finally { + Table.cleanup(); + } + }); + + it('arms system tables too, which databases does not enumerate', () => { + const systemTables = Object.values(databases.system); + assert(systemTables.length > 0); + assert(!Object.keys(databases).includes('system'), 'the system database is enumerable now; simplify this test'); + const armed = new Set(); + const originals = systemTables.map((Table) => [Table, Table._armCleanupScan]); + for (const Table of systemTables) Table._armCleanupScan = () => armed.add(Table); + try { + armTableCleanupScans(); + } finally { + for (const [Table, original] of originals) Table._armCleanupScan = original; + } + assert.strictEqual(armed.size, systemTables.length); + }); });