diff --git a/dataLayer/DESIGN.md b/dataLayer/DESIGN.md
index 6062e930d0..481420d215 100644
--- a/dataLayer/DESIGN.md
+++ b/dataLayer/DESIGN.md
@@ -42,7 +42,7 @@ Set `audit = true` on the directive's `CreateTableObject`: `createTables()` audi
- A node that never runs the owner's code path never declares the rest, and its replicated rows are never removed.
- An install on a pre-release of the directive's version never runs it (the `compareVersions` caveat).
-Repeating a local `table()` declaration repairs any of these in place, keeping every row, and is a no-op once the shape matches. Storage-layer limits: a thread that only loads the table scans daily, so a spent row can stay a day (reads skip it); under `threads: 0` nothing scans until the first expiring write or declaration after restart; and a dropped attribute's index store stays on disk, since `table()` looks it up by table name. The certificate verification tables follow the same pattern through `security/certificateVerification/verificationTables.ts`; their rows are covered in security/DESIGN.md.
+Repeating a local `table()` declaration repairs any of these in place, keeping every row, and is a no-op once the shape matches. Storage-layer limit: a dropped attribute's index store stays on disk, since `table()` looks it up by table name. The certificate verification tables follow the same pattern through `security/certificateVerification/verificationTables.ts`; their rows are covered in security/DESIGN.md.
## RocksDB restore: the restore lock + marker protocol (`dataLayer/restoreMarker.ts`)
diff --git a/dataLayer/harperBridge/ResourceBridge.ts b/dataLayer/harperBridge/ResourceBridge.ts
index 0dc8da27e0..d014b4e78c 100644
--- a/dataLayer/harperBridge/ResourceBridge.ts
+++ b/dataLayer/harperBridge/ResourceBridge.ts
@@ -168,6 +168,7 @@ export class ResourceBridge extends BridgeMethods {
attributes,
schemaDefined,
expiration: tableCreateObj.expiration,
+ eviction: tableCreateObj.eviction,
audit: tableCreateObj.audit,
});
}
diff --git a/dataLayer/schemaDescribe.ts b/dataLayer/schemaDescribe.ts
index 909e94e66c..64bf24280e 100644
--- a/dataLayer/schemaDescribe.ts
+++ b/dataLayer/schemaDescribe.ts
@@ -222,7 +222,7 @@ async function descTable(describeTableObject: any, attrPerms?: any) {
db_size,
};
if (tableObj.replicate !== undefined) tableResult.replicate = tableObj.replicate;
- if (tableObj.expirationMS !== undefined) tableResult.expiration = tableObj.expirationMS / 1000 + 's';
+ if (tableObj.expirationMS || tableObj.evictionMS) tableResult.expiration = tableObj.expirationMS / 1000 + 's';
if (tableObj.sealed !== undefined) tableResult.sealed = tableObj.sealed;
if (tableObj.cacheControl != null) tableResult.cacheControl = tableObj.cacheControl;
if ((tableObj as any).sources?.length > 0)
diff --git a/docs/reference/database/schema.md b/docs/reference/database/schema.md
index c89f0428e3..62cf232078 100644
--- a/docs/reference/database/schema.md
+++ b/docs/reference/database/schema.md
@@ -107,6 +107,12 @@ type WeatherCache @table(expiration: 300, eviction: 3300, scanInterval: 600) {
}
```
+**Changing or removing them**
+
+`expiration` and `eviction` are saved with the table, and a restart applies what the schema last declared. `@table` (and `defineTable`) sets each of the two that it names and clears each one it omits, including a value `create_table` saved; setting one to `0` clears it too. Without `expiration`, records written afterwards no longer expire; without `eviction`, they are removed as soon as they expire. Records already written keep the expiry they were given. Before v5.4.0 an omitted value stayed in effect until the table was dropped. A TTL that application code sets at runtime (for example through `sourcedFrom` options) is not cleared by a schema that declares none or sets it to `0`; a TTL the schema itself set later is.
+
+Each value must be a number of seconds, zero or more. A negative or non-numeric value fails the declaration with an error naming the table and the argument, and the table is left unchanged.
+
#### How `scanInterval` Determines the Eviction Cycle
`scanInterval` determines fixed clock-aligned times when eviction runs. Harper divides the clock into evenly spaced anchors based on the interval, calculated in the server's local timezone. As a result:
diff --git a/docs/reference/operations-api/operations.md b/docs/reference/operations-api/operations.md
index 6ba366e16f..7520668a2d 100644
--- a/docs/reference/operations-api/operations.md
+++ b/docs/reference/operations-api/operations.md
@@ -175,7 +175,7 @@ Drops a database and all its tables/records. **Replicates to all cluster nodes b
### `create_table`
-Creates a new table. Optional fields: `database` (defaults to `data`), `attributes` (array defining schema), `expiration` (TTL in seconds).
+Creates a new table. Optional fields: `database` (defaults to `data`), `attributes` (array defining schema), `expiration` (TTL in seconds), `eviction` (seconds after `expiration` before a record is removed; see [`@table`](../database/schema.md#table)). A negative or non-numeric `expiration` or `eviction` is rejected with a 400 error.
```json
{
diff --git a/docs/release-notes/v5-lincoln/5.4/removing-a-table-expiration-clears-it.md b/docs/release-notes/v5-lincoln/5.4/removing-a-table-expiration-clears-it.md
new file mode 100644
index 0000000000..f4a664eea6
--- /dev/null
+++ b/docs/release-notes/v5-lincoln/5.4/removing-a-table-expiration-clears-it.md
@@ -0,0 +1,7 @@
+---
+section: 'Storage'
+---
+
+### Removing a Table's Expiration Clears It
+
+`@table` (and `defineTable`) now clears an `expiration` or `eviction` it no longer names, or sets to `0`, after a restart and on threads that load the table without declaring it. Before, the old value stayed in effect, so a table kept expiring records after its schema stopped declaring a TTL. If you relied on that, declare the value again. Records already written keep the expiry they were given. A negative or non-numeric `expiration`, `eviction` or `scanInterval` now fails the declaration with an error naming the table and argument, instead of failing the table later; `create_table` accepts `eviction` and rejects a negative or non-numeric `expiration` or `eviction` with a 400. Expired records are now removed on schedule after a restart with `threads.count: 0`, and on a table that has only `eviction`, without waiting for a write. See [`@table`](/reference/v5/database/schema#table).
diff --git a/integrationTests/database/table-expiration-config-negative/config.yaml b/integrationTests/database/table-expiration-config-negative/config.yaml
new file mode 100644
index 0000000000..bb034a091d
--- /dev/null
+++ b/integrationTests/database/table-expiration-config-negative/config.yaml
@@ -0,0 +1,5 @@
+rest: true
+graphqlSchema:
+ files: 'schema.graphql'
+ # Shortened from the 30s default so a run that times out instead of failing fast costs seconds.
+ timeout: 5000
diff --git a/integrationTests/database/table-expiration-config-negative/schema.graphql b/integrationTests/database/table-expiration-config-negative/schema.graphql
new file mode 100644
index 0000000000..7dcc9649cc
--- /dev/null
+++ b/integrationTests/database/table-expiration-config-negative/schema.graphql
@@ -0,0 +1,4 @@
+type NegativeTtl @table(database: "ttl_config_bad", expiration: -1) @export {
+ id: ID @primaryKey
+ value: String
+}
diff --git a/integrationTests/database/table-expiration-config.test.ts b/integrationTests/database/table-expiration-config.test.ts
new file mode 100644
index 0000000000..b0bdfd1b96
--- /dev/null
+++ b/integrationTests/database/table-expiration-config.test.ts
@@ -0,0 +1,405 @@
+/**
+ * A table's TTL configuration on every thread that holds the table: the threads that declare it, the threads
+ * that only load it from the catalog, and the main thread under `threads: 0`, across restarts, on RocksDB and
+ * LMDB (harper#1286, #2831, #3372, #3934, #3935).
+ *
+ * The cleanup scan runs on the store-maintenance owner, the last worker. A table created by `create_table`, or
+ * declared through the fixture's TtlControl endpoint on another worker, reaches the owner only through the
+ * catalog. Under `threads: 0` the main thread loads every table before it becomes worker 0.
+ *
+ * Expiration and eviction are 2 s each (8 s on the tables that live across a restart), so the derived scan
+ * interval is (2 + 2) / 4 = 1 s (4 s).
+ *
+ * Reproduction:
+ * npm run build && npm run test:integration -- "integrationTests/database/table-expiration-config.test.ts"
+ */
+import { suite, test, before, after } from 'node:test';
+import { ok, strictEqual } from 'node:assert';
+import { request as httpRequest } from 'node:http';
+import { join, resolve } from 'node:path';
+import { cp, mkdtemp, readFile, writeFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { setTimeout as sleep } from 'node:timers/promises';
+import { startHarper, killHarper, teardownHarper, type ContextWithHarper } from '@harperfast/integration-testing';
+
+const FIXTURE = 'table-expiration-config';
+// A test reaches a chosen worker by retrying on fresh connections, which needs connections spread across
+// workers: Windows serves HTTP from one worker, and Bun does not spread them.
+const skipWorkerPinning = process.platform === 'win32' || process.env.HARPER_RUNTIME === 'bun';
+const NEGATIVE_FIXTURE = 'table-expiration-config-negative';
+const DATABASE = 'ttl_config';
+const EXPIRATION = 2;
+const EVICTION = 2;
+const ROWS = 5;
+// Tables that live across a restart use a longer TTL, so their rows only become evictable after the restart and
+// only the restarted process can reclaim them. Derived scan interval (8 + 8) / 4 = 4 s.
+const RUNTIME_EXPIRATION = 8;
+const RUNTIME_EVICTION = 8;
+const RECLAIM_MARGIN_MS = 15_000;
+// past expiration + eviction plus three scans, so a TTL still in effect would have hidden and reclaimed the rows
+const CLEARED_HOLD_MS = (EXPIRATION + EVICTION) * 1000 + 3_000;
+
+interface OperationResult {
+ status: number;
+ body: any;
+}
+
+function parseIfJson(text: string): any {
+ try {
+ return JSON.parse(text);
+ } catch {
+ return text;
+ }
+}
+
+async function operation(ctx: ContextWithHarper, body: Record): Promise {
+ const response = await fetch(ctx.harper.operationsAPIURL, {
+ method: 'POST',
+ headers: {
+ 'Content-Type': 'application/json',
+ 'Authorization': basicAuth(ctx),
+ },
+ body: JSON.stringify(body),
+ });
+ const text = await response.text();
+ return { status: response.status, body: parseIfJson(text) };
+}
+
+async function ok200(ctx: ContextWithHarper, body: Record) {
+ const result = await operation(ctx, body);
+ strictEqual(result.status, 200, `${body.operation} failed: ${JSON.stringify(result.body)}`);
+ return result.body;
+}
+
+// describe_table counts stored entries without a query or GET, so it neither hides nor touches expired rows
+async function storedRows(ctx: ContextWithHarper, table: string): Promise {
+ return (await ok200(ctx, { operation: 'describe_table', database: DATABASE, table })).record_count;
+}
+
+async function waitForReclaim(ctx: ContextWithHarper, table: string, dueAt: number, why: string) {
+ const deadline = dueAt + RECLAIM_MARGIN_MS;
+ let count: number;
+ do {
+ await sleep(500);
+ count = await storedRows(ctx, table);
+ } while (count > 0 && Date.now() < deadline);
+ strictEqual(
+ count,
+ 0,
+ `${count} rows of ${table} still stored ${RECLAIM_MARGIN_MS / 1000}s after they were due: ${why}`
+ );
+}
+
+async function insertRows(ctx: ContextWithHarper, table: string) {
+ const records = Array.from({ length: ROWS }, (_, i) => ({ id: String(i), value: `v${i}` }));
+ await ok200(ctx, { operation: 'insert', database: DATABASE, table, records });
+}
+
+const basicAuth = (ctx: ContextWithHarper) =>
+ `Basic ${Buffer.from(`${ctx.harper.admin.username}:${ctx.harper.admin.password}`).toString('base64')}`;
+
+// a fresh connection per request, so retries reach different workers
+function http(
+ ctx: ContextWithHarper,
+ method: string,
+ path: string,
+ body?: Record
+): Promise<{ status: number; text: string }> {
+ const payload = body === undefined ? undefined : JSON.stringify(body);
+ return new Promise((resolveRequest, reject) => {
+ const req = httpRequest(
+ new URL(path, ctx.harper.httpURL),
+ {
+ method,
+ agent: false,
+ headers: {
+ Authorization: basicAuth(ctx),
+ ...(payload === undefined
+ ? {}
+ : { 'Content-Type': 'application/json', 'Content-Length': Buffer.byteLength(payload) }),
+ },
+ },
+ (res) => {
+ const chunks: Buffer[] = [];
+ res.on('data', (chunk) => chunks.push(chunk));
+ res.on('end', () => resolveRequest({ status: res.statusCode ?? 0, text: Buffer.concat(chunks).toString() }));
+ }
+ );
+ req.setTimeout(30_000, () => req.destroy(new Error(`${method} ${path}: no response`)));
+ req.on('error', reject);
+ req.end(payload);
+ });
+}
+
+async function control(ctx: ContextWithHarper, body: Record): Promise {
+ const { status, text } = await http(ctx, 'POST', '/TtlControl/', body);
+ if (status !== 200) throw new Error(`TtlControl ${body.op}: ${status} ${text}`);
+ return JSON.parse(text);
+}
+
+async function untilServed(ctx: ContextWithHarper, body: Record) {
+ const deadline = Date.now() + 60_000;
+ for (;;) {
+ const result = await control(ctx, body);
+ if (!result.wrongThread) return result;
+ if (Date.now() > deadline) throw new Error(`TtlControl ${body.op} never reached a worker that could serve it`);
+ await sleep(10);
+ }
+}
+
+const workerTtl = (ctx: ContextWithHarper, table: string, worker: number) =>
+ untilServed(ctx, { op: 'ttl', table, onWorker: worker });
+
+async function waitForWorkerTtl(
+ ctx: ContextWithHarper,
+ table: string,
+ worker: number,
+ expected: { expirationMS: number; evictionMS: number }
+) {
+ const deadline = Date.now() + 20_000;
+ let seen: any;
+ do {
+ seen = await workerTtl(ctx, table, worker);
+ if (seen.loaded && seen.expirationMS === expected.expirationMS && seen.evictionMS === expected.evictionMS) return;
+ await sleep(200);
+ } while (Date.now() < deadline);
+ throw new Error(`worker ${worker} holds ${JSON.stringify(seen)} for ${table}, expected ${JSON.stringify(expected)}`);
+}
+
+async function assertRowsKept(ctx: ContextWithHarper, table: string, why: string) {
+ await sleep(CLEARED_HOLD_MS);
+ strictEqual(await storedRows(ctx, table), ROWS, `rows of ${table} were reclaimed on: ${why}`);
+ for (let i = 0; i < ROWS * 4; i++) {
+ const { status } = await http(ctx, 'GET', `/${table}/${i % ROWS}`);
+ strictEqual(status, 200, `GET /${table}/${i % ROWS} was hidden as expired on: ${why}`);
+ }
+}
+
+function softChecks() {
+ const failures: string[] = [];
+ return {
+ async check(run: () => Promise) {
+ try {
+ await run();
+ } catch (error) {
+ failures.push((error as Error).message);
+ }
+ },
+ assertNone() {
+ strictEqual(failures.length, 0, `${failures.length} checks failed:\n- ${failures.join('\n- ')}`);
+ },
+ };
+}
+
+async function stage(fixtures: string[]) {
+ const dataRootDir = await mkdtemp(
+ join(process.env.HARPER_INTEGRATION_TEST_INSTALL_PARENT_DIR || tmpdir(), 'harper-integration-test-')
+ );
+ for (const fixture of fixtures)
+ await cp(resolve(import.meta.dirname, fixture), join(dataRootDir, 'components', fixture), {
+ recursive: true,
+ dereference: true,
+ });
+ return dataRootDir;
+}
+
+async function dropDeclaredTtl(ctx: ContextWithHarper) {
+ const schemaPath = join(ctx.harper.dataRootDir, 'components', FIXTURE, 'schema.graphql');
+ const schema = await readFile(schemaPath, 'utf8');
+ const cleared = schema.replace(`, expiration: ${EXPIRATION}, eviction: ${EVICTION}`, '');
+ ok(cleared !== schema, 'the fixture schema no longer declares the TTL this test removes');
+ await writeFile(schemaPath, cleared);
+}
+
+async function createRuntimeTtl(ctx: ContextWithHarper) {
+ await ok200(ctx, {
+ operation: 'create_table',
+ database: DATABASE,
+ table: 'RuntimeTtl',
+ primary_key: 'id',
+ expiration: RUNTIME_EXPIRATION,
+ eviction: RUNTIME_EVICTION,
+ });
+}
+
+async function restartWithoutDeclaredTtl(ctx: ContextWithHarper, options: Record, dueAt: number) {
+ await killHarper(ctx);
+ await dropDeclaredTtl(ctx);
+ await startHarper(ctx, options);
+ ok(Date.now() < dueAt, 'the restart outlasted the runtime TTL, so it cannot show which process reclaimed the rows');
+}
+
+async function assertDeclaredTtlRemoved(ctx: ContextWithHarper) {
+ const described = await ok200(ctx, { operation: 'describe_table', database: DATABASE, table: 'Declared' });
+ strictEqual(described.expiration, undefined, 'describe_table (the main thread) still reports the removed expiration');
+ await insertRows(ctx, 'Declared');
+ await assertRowsKept(ctx, 'Declared', 'a schema whose TTL was removed, after a restart');
+}
+
+for (const engine of ['rocksdb', 'lmdb']) {
+ const env = engine === 'lmdb' ? { HARPER_STORAGE_ENGINE: 'lmdb' } : {};
+
+ suite(`Table TTL configuration with four workers (${engine})`, (ctx: ContextWithHarper) => {
+ const THREADS = 4;
+ const OWNER = THREADS - 1;
+ const options = { config: { threads: { count: THREADS } }, env };
+
+ before(async () => {
+ ctx.harper = { dataRootDir: await stage([FIXTURE, NEGATIVE_FIXTURE]) } as any;
+ await startHarper(ctx, options);
+ });
+
+ after(async () => {
+ await teardownHarper(ctx);
+ });
+
+ test('a negative TTL fails where it is declared, naming the table and field (#1286, #3935)', async () => {
+ const name = `${NEGATIVE_FIXTURE}.graphqlSchema`;
+ const deadline = Date.now() + 30_000;
+ let entry: any;
+ do {
+ const status = await ok200(ctx, { operation: 'get_status' });
+ entry = status.componentStatus?.find((candidate: any) => candidate.name === name);
+ if (entry?.status === 'error') break;
+ await sleep(250);
+ } while (Date.now() < deadline);
+ strictEqual(
+ entry?.status,
+ 'error',
+ `@table(expiration: -1) did not fail its component: ${JSON.stringify(entry)}`
+ );
+ const reported = JSON.stringify(entry);
+ ok(!/timed out/i.test(reported), `the component failed on a timeout instead of the TTL: ${reported}`);
+ ok(
+ /expiration/.test(reported) && reported.includes('ttl_config_bad.NegativeTtl'),
+ `the failure does not name the table and field: ${reported}`
+ );
+
+ for (const [field, table] of [
+ ['eviction', 'NegativeEviction'],
+ ['expiration', 'NegativeExpiration'],
+ ]) {
+ const created = await operation(ctx, {
+ operation: 'create_table',
+ database: DATABASE,
+ table,
+ primary_key: 'id',
+ [field]: -1,
+ });
+ strictEqual(created.status, 400, `create_table with ${field}: -1 returned ${JSON.stringify(created)}`);
+ const error = JSON.stringify(created.body);
+ ok(
+ error.includes(field) && error.includes(`${DATABASE}.${table}`),
+ `create_table does not name the table and field: ${error}`
+ );
+ const described = await operation(ctx, { operation: 'describe_table', database: DATABASE, table });
+ ok(described.status !== 200, `a rejected create_table left ${table} behind`);
+ }
+ });
+
+ test('the declaring thread reclaims expired rows on the derived cadence', { timeout: 60_000 }, async () => {
+ await insertRows(ctx, 'Declared');
+ await waitForReclaim(ctx, 'Declared', Date.now() + (EXPIRATION + EVICTION) * 1000, 'a schema-declared TTL');
+ });
+
+ test(
+ 'an eviction-only TTL adopted from the catalog reclaims rows (#3935)',
+ { timeout: 120_000, skip: skipWorkerPinning },
+ async () => {
+ const table = 'AdoptedEviction';
+ const { workerIndex } = await untilServed(ctx, { op: 'declare', table, avoidWorker: OWNER });
+ // loaded before it gains a TTL, so the owner adopts the TTL instead of loading the table fresh with it
+ await waitForWorkerTtl(ctx, table, OWNER, { expirationMS: 0, evictionMS: 0 });
+ await untilServed(ctx, { op: 'declare', table, eviction: EVICTION, onWorker: workerIndex });
+ await waitForWorkerTtl(ctx, table, OWNER, { expirationMS: 0, evictionMS: EVICTION * 1000 });
+ await untilServed(ctx, { op: 'write', table, count: ROWS, expiresInMs: 500 });
+ await waitForReclaim(ctx, table, Date.now() + 500 + EVICTION * 1000, 'an eviction-only TTL the owner adopted');
+ }
+ );
+
+ // both declarations on one worker: a worker keeps a TTL its own declaration set, as a restart re-applies it
+ test(
+ 'a re-declaration without a TTL clears it on every worker (#3934)',
+ { timeout: 120_000, skip: skipWorkerPinning },
+ async () => {
+ const table = 'AdoptedClear';
+ const declared = { expirationMS: EXPIRATION * 1000, evictionMS: EVICTION * 1000 };
+ const { workerIndex } = await untilServed(ctx, {
+ op: 'declare',
+ table,
+ expiration: EXPIRATION,
+ eviction: EVICTION,
+ avoidWorker: OWNER,
+ });
+ for (let worker = 0; worker < THREADS; worker++) await waitForWorkerTtl(ctx, table, worker, declared);
+ await untilServed(ctx, { op: 'declare', table, onWorker: workerIndex });
+ const checks = softChecks();
+ for (let worker = 0; worker < THREADS; worker++)
+ await checks.check(() => waitForWorkerTtl(ctx, table, worker, { expirationMS: 0, evictionMS: 0 }));
+ checks.assertNone();
+ }
+ );
+
+ test(
+ 'after a restart: a runtime table is reclaimed without a write, and a removed @table TTL stays removed (#3372, #3934)',
+ { timeout: 180_000, skip: skipWorkerPinning },
+ async () => {
+ const checks = softChecks();
+ await createRuntimeTtl(ctx);
+ await insertRows(ctx, 'RuntimeTtl');
+ const dueAt = Date.now() + (RUNTIME_EXPIRATION + RUNTIME_EVICTION) * 1000;
+ await restartWithoutDeclaredTtl(ctx, options, dueAt);
+
+ await checks.check(() => waitForReclaim(ctx, 'RuntimeTtl', dueAt, 'a runtime-created table, no write'));
+ for (let worker = 0; worker < THREADS; worker++)
+ await checks.check(() => waitForWorkerTtl(ctx, 'Declared', worker, { expirationMS: 0, evictionMS: 0 }));
+ await checks.check(() => assertDeclaredTtlRemoved(ctx));
+ checks.assertNone();
+ }
+ );
+ });
+
+ suite(`Table TTL configuration with threads: 0 (${engine})`, (ctx: ContextWithHarper) => {
+ const options = { config: { threads: { count: 0 } }, env };
+
+ before(async () => {
+ ctx.harper = { dataRootDir: await stage([FIXTURE]) } as any;
+ await startHarper(ctx, options);
+ });
+
+ after(async () => {
+ await teardownHarper(ctx);
+ });
+
+ test(
+ 'after a restart, the main thread reclaims catalog-loaded tables without a write and keeps a removed TTL removed (#2831, #3934, #3935)',
+ { timeout: 180_000 },
+ async () => {
+ const checks = softChecks();
+ strictEqual((await ok200(ctx, { operation: 'get_configuration' })).threads.count, 0);
+ await createRuntimeTtl(ctx);
+ await ok200(ctx, {
+ operation: 'create_table',
+ database: DATABASE,
+ table: 'RuntimeEviction',
+ primary_key: 'id',
+ eviction: RUNTIME_EVICTION,
+ });
+ await checks.check(() =>
+ waitForWorkerTtl(ctx, 'RuntimeEviction', 0, { expirationMS: 0, evictionMS: RUNTIME_EVICTION * 1000 })
+ );
+ await insertRows(ctx, 'RuntimeTtl');
+ const expiresInMs = RUNTIME_EXPIRATION * 1000;
+ await untilServed(ctx, { op: 'write', table: 'RuntimeEviction', count: ROWS, expiresInMs });
+ const dueAt = Date.now() + (RUNTIME_EXPIRATION + RUNTIME_EVICTION) * 1000;
+ await restartWithoutDeclaredTtl(ctx, options, dueAt);
+
+ await checks.check(() => waitForReclaim(ctx, 'RuntimeTtl', dueAt, 'a runtime-created table, no write'));
+ await checks.check(() => waitForReclaim(ctx, 'RuntimeEviction', dueAt, 'an eviction-only runtime table'));
+ await checks.check(() => waitForWorkerTtl(ctx, 'Declared', 0, { expirationMS: 0, evictionMS: 0 }));
+ await checks.check(() => assertDeclaredTtlRemoved(ctx));
+ checks.assertNone();
+ }
+ );
+ });
+}
diff --git a/integrationTests/database/table-expiration-config/config.yaml b/integrationTests/database/table-expiration-config/config.yaml
new file mode 100644
index 0000000000..41005f90af
--- /dev/null
+++ b/integrationTests/database/table-expiration-config/config.yaml
@@ -0,0 +1,5 @@
+rest: true
+graphqlSchema:
+ files: 'schema.graphql'
+jsResource:
+ files: 'resources.js'
diff --git a/integrationTests/database/table-expiration-config/resources.js b/integrationTests/database/table-expiration-config/resources.js
new file mode 100644
index 0000000000..479eeaf253
--- /dev/null
+++ b/integrationTests/database/table-expiration-config/resources.js
@@ -0,0 +1,47 @@
+// POST /TtlControl/ drives table-expiration-config.test.ts:
+// { op: 'declare', table, expiration?, eviction?, avoidWorker?, onWorker? } — defineTable on the worker that
+// serves the request; a worker that avoidWorker excludes, or onWorker does not name, answers { wrongThread }
+// and the test retries on a fresh connection, so the scan owner only ever learns the TTL from the catalog
+// { op: 'ttl', table, onWorker } — that worker's view of the table: whether it is loaded, and its TTL in ms
+// { op: 'write', table, count, expiresInMs } — rows whose own expiry is expiresInMs from now
+import { Resource, databases, defineTable, types } from 'harper';
+import { workerData } from 'node:worker_threads';
+
+const DATABASE = 'ttl_config';
+const workerIndex = () => workerData?.workerIndex ?? 0;
+
+export class TtlControl extends Resource {
+ static loadAsInstance = false;
+ async post(_target, body) {
+ const { op, table } = body;
+ if (op === 'declare') {
+ if (
+ (body.avoidWorker !== undefined && workerIndex() === body.avoidWorker) ||
+ (body.onWorker !== undefined && workerIndex() !== body.onWorker)
+ )
+ return { wrongThread: true, workerIndex: workerIndex() };
+ const options = { database: DATABASE };
+ if (body.expiration !== undefined) options.expiration = body.expiration;
+ if (body.eviction !== undefined) options.eviction = body.eviction;
+ defineTable(table, { id: types.id.primaryKey, value: types.string.nullable }, options);
+ return { workerIndex: workerIndex() };
+ }
+ if (op === 'ttl') {
+ if (workerIndex() !== body.onWorker) return { wrongThread: true, workerIndex: workerIndex() };
+ const Table = databases[DATABASE]?.[table];
+ return {
+ workerIndex: workerIndex(),
+ loaded: Boolean(Table),
+ expirationMS: Table?.expirationMS || 0,
+ evictionMS: Table?.evictionMS || 0,
+ };
+ }
+ if (op === 'write') {
+ const Table = databases[DATABASE][table];
+ const expiresAt = Date.now() + body.expiresInMs;
+ for (let i = 0; i < body.count; i++) await Table.put(String(i), { id: String(i), value: 'v' }, { expiresAt });
+ return { workerIndex: workerIndex(), expiresAt };
+ }
+ throw new Error(`unknown op ${op}`);
+ }
+}
diff --git a/integrationTests/database/table-expiration-config/schema.graphql b/integrationTests/database/table-expiration-config/schema.graphql
new file mode 100644
index 0000000000..a0347c5115
--- /dev/null
+++ b/integrationTests/database/table-expiration-config/schema.graphql
@@ -0,0 +1,5 @@
+# The test rewrites this file between restarts to drop the TTL arguments.
+type Declared @table(database: "ttl_config", expiration: 2, eviction: 2) @export {
+ id: ID @primaryKey
+ value: String
+}
diff --git a/resources/DESIGN.md b/resources/DESIGN.md
index 78e0a7f892..d746c0b529 100644
--- a/resources/DESIGN.md
+++ b/resources/DESIGN.md
@@ -269,7 +269,7 @@ If this same write (origin and log key) emptied the key, `_writeDelete` writes n
## Cluster-origin table definitions are additive-only
-An `origin: 'cluster'` definition is a peer snapshot, so `table()` only adds from it and logs what it discards. Drops and redefinitions never converge via gossip (recover via local `create_attribute` or redeploy). Abandoned index builds still recover (`clusterSchemaMerge.test.js`). A reload keeps a thread's own TTL.
+An `origin: 'cluster'` definition is a peer snapshot, so `table()` only adds from it and logs what it discards. Drops and redefinitions never converge via gossip (recover via local `create_attribute` or redeploy). Abandoned index builds still recover (`clusterSchemaMerge.test.js`). A reload keeps a thread's own TTL. `@table` omitting a TTL clears it (`null`); `table()` omitting one keeps it.
## `drop_attribute` never removes the primary key, and a strip keeps each row's version
diff --git a/resources/MAP.md b/resources/MAP.md
index 4f2cf90865..5533f135af 100644
--- a/resources/MAP.md
+++ b/resources/MAP.md
@@ -46,7 +46,7 @@ Where things are in `resources/`. The rules live in [DESIGN.md](DESIGN.md): read
- response shape (select) → `transformEntryForSelect` in Table.ts; `transformForSelect` in Resource.ts
- selects answered from index entries alone → `coveredSelectAttribute` in Table.ts; `coverIndexEntries` in search.ts
- sorting → `transformToOrderedSelect` in Table.ts; whether the lead already supplies the order → `scansInAttributeOrder` in search.ts
-- record TTL and eviction → `setTTLExpiration`, `_adoptCatalogExpiration`, `evict`, `scheduleCleanup` in Table.ts
+- record TTL and eviction → `setTTLExpiration`, `_adoptCatalogExpiration`, `evict`, `scheduleCleanup` in Table.ts; `armTableCleanupScans` in databases.ts
- cache miss to source → `getFromSource` in Table.ts
- subscriptions and replay → `subscribe`, `eventFromAudit` in Table.ts; `addSubscription` in transactionBroadcast.ts
- replication apply loop → `sourcedFrom` in Table.ts
diff --git a/resources/Table.ts b/resources/Table.ts
index 61de4089e7..150fc52f56 100644
--- a/resources/Table.ts
+++ b/resources/Table.ts
@@ -811,6 +811,36 @@ export function replicaExpirationMsOf(seconds: unknown): number | undefined {
if (!Number.isFinite(ms)) throw new Error('replicaExpiration is too large');
return ms;
}
+// the scan cadence of a table with no TTL, which an expiring write arms for its per-record expiry
+const DEFAULT_CLEANUP_INTERVAL = 86_400_000;
+// the latest Date, in seconds: an expiry past it cannot be stamped on a record
+const MAX_TTL_SECONDS = 8.64e12;
+/**
+ * A table's `expiration`, `eviction` or `scanInterval` in seconds, as a declaration or `setTTLExpiration` gives
+ * it: undefined when not given, 0 for `null` (cleared), else a number or a numeric string (a create persisted it
+ * as given) that is finite and zero or more.
+ */
+export function ttlSecondsOf(table: string, field: string, value: unknown): number | undefined {
+ if (value === undefined) return undefined;
+ if (value === null) return 0;
+ const seconds =
+ typeof value === 'number' ? value : typeof value === 'string' && value.trim() !== '' ? Number(value) : NaN;
+ if (seconds >= 0 && seconds <= MAX_TTL_SECONDS) return seconds;
+ throw new ClientError(
+ `${field} of '${table}' must be a non-negative number of seconds, got ${describeTtlValue(value)}`
+ );
+}
+/** A TTL field as the catalog holds it: absent unless a valid positive number of seconds. */
+export function savedTtlSeconds(value: unknown): number | undefined {
+ const seconds = typeof value === 'number' || typeof value === 'string' ? Number(value) : NaN;
+ return seconds > 0 && seconds <= MAX_TTL_SECONDS ? seconds : undefined;
+}
+/** A rejected TTL value for a message: bounded, and never a value that cannot be rendered. */
+export function describeTtlValue(value: unknown): string {
+ if (typeof value === 'number' || typeof value === 'bigint' || typeof value === 'boolean') return String(value);
+ if (typeof value === 'string') return JSON.stringify(value.slice(0, 40));
+ return value === null ? 'null' : typeof value;
+}
const TEST_WRITE_KEY_BUFFER = Buffer.allocUnsafeSlow(8192);
const MAX_KEY_BYTES = 1978;
const EVENT_HIGH_WATER_MARK = 100;
@@ -1195,7 +1225,8 @@ interface TableResourceClass {
getResidencyById: (id: Id) => number | void;
readonly residency: Readonly | undefined;
_setResidencyPolicy(policy: ResidencyPolicy): void;
- _adoptCatalogExpiration(expiration?: number, eviction?: number): void;
+ _adoptCatalogExpiration(expiration?: number, eviction?: number, expiryScan?: boolean): void;
+ _armCleanupScan(): void;
getResidencySlot(id: Id, record?: object): number;
getRecordResidency(entry: Partial): readonly string[] | undefined;
get expirationMS(): any;
@@ -1964,13 +1995,15 @@ export function makeTable(options): TableResourceClass {
// a store on any thread wakes the sweep when [1] is 0
let replicaSweepState: Int32Array | undefined;
let recordScanInterval = 0;
- let recordCleanupInterval = 86400000;
const replicaRefreshInFlight = new Set();
const replicaRefreshInFlightComposite = new Set();
// Set when the TTL exists only on this thread: either application code configured it at runtime, or
// an isolated application's schema was declared here. Hydrating persisted metadata does not set it:
// dedicated workers open unrelated shared tables too, whose scan remains owned by the pool.
- let ttlConfiguredByApplication = false;
+ let ttlOwnedByThread = false;
+ // Which part of the TTL in effect application code set, rather than a schema: a declaration with no
+ // TTL keeps that part and clears the rest. The number form sets only the expiration.
+ let applicationTtl: 'none' | 'expiration' | 'all' = 'none';
let ttlDeclaredBySchema = false;
let ttlFromLoad = false; // true only around a catalog load's setTTLExpiration call
evictionMs ??= 0;
@@ -2036,7 +2069,7 @@ export function makeTable(options): TableResourceClass {
let prefetchCallbacks = [];
let untilNextPrefetch = 1;
let nonPrefetchSequence = 2;
- let cleanupInterval = 86400000;
+ let cleanupInterval = DEFAULT_CLEANUP_INTERVAL;
let cleanupPriority = 0;
let lastCleanupInterval: number | undefined;
let cleanupTimer: NodeJS.Timeout | undefined;
@@ -2046,6 +2079,8 @@ export function makeTable(options): TableResourceClass {
let disposed = false;
// true once a table-level expiration/eviction/scanInterval has armed the periodic cleanup scan at setup
let expirationScanScheduled = false;
+ // rows stamped under a TTL since cleared keep their expiry, so their (daily) scan outlives the TTL
+ let retainExpiryScan = Boolean(options.expiryScan);
// set on the first expiring write so the unscheduled-expiration warning is evaluated at most once per table
let expirationWarningChecked = false;
let writerResidencyWarned = false;
@@ -2770,24 +2805,30 @@ export function makeTable(options): TableResourceClass {
* load would. A TTL that this thread's own schema declaration or application code set is kept, as it is
* across a restart, which re-applies it over the catalog.
*/
- static _adoptCatalogExpiration(expiration?: number, eviction?: number) {
- if (ttlConfiguredByApplication || ttlDeclaredBySchema) return;
- const loadedExpirationMs = expiration && expiration > 0 ? expiration * 1000 : 0;
- const loadedEvictionMs = eviction && eviction > 0 ? eviction * 1000 : 0;
- if (!loadedExpirationMs && !loadedEvictionMs) return;
- if (loadedExpirationMs === expirationMs && loadedEvictionMs === evictionMs) return;
- evictionMs = loadedEvictionMs;
- if (!loadedExpirationMs) {
- expirationMs = 0;
- return;
- }
+ static _adoptCatalogExpiration(expiration?: number, eviction?: number, expiryScan?: boolean) {
+ // rows another thread stamped under a TTL it since cleared need the scan whoever owns this thread's TTL
+ if (expiryScan && !retainExpiryScan) {
+ retainExpiryScan = true;
+ scheduleCleanup();
+ }
+ if (ttlOwnedByThread || ttlDeclaredBySchema) return;
+ expiration ||= 0;
+ eviction ||= 0;
+ if (expiration * 1000 === (expirationMs || 0) && eviction * 1000 === evictionMs) return;
ttlFromLoad = true;
try {
- TableResource.setTTLExpiration(loadedExpirationMs / 1000);
+ TableResource.setTTLExpiration({ expiration, eviction });
} finally {
ttlFromLoad = false;
}
}
+ /**
+ * Arm the cleanup scan of a table this thread loaded before it could own the scan: under `threads: 0` the
+ * main thread loads every table before it becomes worker 0.
+ */
+ static _armCleanupScan() {
+ if (expirationScanScheduled || evictionMs || retainExpiryScan) scheduleCleanup();
+ }
static getResidencySlot(id: Id, record?: object): number {
return calculateResidencySlot(id, record);
}
@@ -3913,9 +3954,11 @@ export function makeTable(options): TableResourceClass {
* @param opts Time in seconds until records expire, or an options object with `expiration`, `eviction`,
* and `scanInterval` (all in seconds, all optional). Number form preserves any previously configured
* eviction/scanInterval; object form replaces all three, unless it names only the replica options. An internal schema ownership-only call with
- * none of those values preserves the settings already loaded from the catalog.
+ * none of those values preserves the settings already loaded from the catalog, and so does an internal
+ * schema call with no TTL on a thread whose application code configured one.
* Either form then scans for expired records every `scanInterval`, or a quarter of expiration + eviction
- * when none is configured.
+ * when none is configured, or daily (for per-record expiry) when the table has no TTL. A rejected value
+ * changes nothing.
*/
static setTTLExpiration(
opts:
@@ -3932,42 +3975,52 @@ export function makeTable(options): TableResourceClass {
) {
if (opts == null || (typeof opts !== 'number' && typeof opts !== 'object'))
throw new Error('Invalid expiration value type');
+ const qualifiedName = `${databaseName}.${tableName}`;
+ const byNumber = typeof opts === 'number';
+ const expirationSeconds = ttlSecondsOf(qualifiedName, 'expiration', byNumber ? opts : opts.expiration);
+ const evictionSeconds = byNumber ? undefined : ttlSecondsOf(qualifiedName, 'eviction', opts.eviction);
+ const scanIntervalSeconds = byNumber ? undefined : ttlSecondsOf(qualifiedName, 'scanInterval', opts.scanInterval);
+ const validatedReplicaExpirationMs = byNumber ? undefined : replicaExpirationMsOf(opts.replicaExpiration);
+ const validatedRefreshFraction = byNumber ? undefined : replicaRefreshFractionOf(opts.replicaRefreshFraction);
const previousReplicaExpirationMs = replicaExpirationMs;
const ownedCleanup = ownsCleanup();
- if (typeof opts === 'object' && opts.replicaExpiration != null) {
- const validated = replicaExpirationMsOf(opts.replicaExpiration);
- replicaExpirationMs = isRocksDB ? validated : 0;
- }
- if (typeof opts === 'object' && opts.replicaRefreshFraction != null)
- replicaRefreshFraction = replicaRefreshFractionOf(opts.replicaRefreshFraction);
- const declaredHere = typeof opts === 'object' && opts.fromSchema;
+ if (validatedReplicaExpirationMs !== undefined)
+ replicaExpirationMs = isRocksDB ? validatedReplicaExpirationMs : 0;
+ if (validatedRefreshFraction !== undefined) replicaRefreshFraction = validatedRefreshFraction;
+ const declaredHere = !byNumber && opts.fromSchema;
const isolatedApplicationOwner = declaredHere && opts.isolatedApplicationOwner;
+ // a schema without a TTL does not clear one this thread's application code configured
+ const keepApplicationTtl =
+ declaredHere && applicationTtl !== 'none' && !expirationSeconds && !evictionSeconds && !scanIntervalSeconds;
const preserveLoadedConfiguration =
- typeof opts === 'object' &&
+ !byNumber &&
(declaredHere || opts.replicaExpiration !== undefined || opts.replicaRefreshFraction !== undefined) &&
- opts.expiration === undefined &&
- opts.eviction === undefined &&
- opts.scanInterval === undefined;
- if (((!ttlFromLoad && !declaredHere) || isolatedApplicationOwner) && !ttlConfiguredByApplication) {
- ttlConfiguredByApplication = true;
+ ((opts.expiration === undefined && opts.eviction === undefined && opts.scanInterval === undefined) ||
+ (keepApplicationTtl && applicationTtl === 'all'));
+ if (((!ttlFromLoad && !declaredHere) || isolatedApplicationOwner) && !ttlOwnedByThread) {
+ ttlOwnedByThread = true;
// the scan owner may have changed with this: re-evaluate even if the interval did not
lastCleanupInterval = undefined;
}
- if (typeof opts === 'number') {
- expirationMs = opts * 1000;
+ if (byNumber) {
+ expirationMs = expirationSeconds * 1000;
} else if (!preserveLoadedConfiguration) {
- // `??` so an explicit 0 is treated as the user's chosen value, not as "missing"
- expirationMs = (opts.expiration ?? 0) * 1000;
- evictionMs = (opts.eviction ?? 0) * 1000;
- recordScanInterval = (opts.scanInterval ?? 0) * 1000;
+ // kept here means the application set only the expiration: the rest is the schema's to clear
+ if (!keepApplicationTtl) expirationMs = (expirationSeconds ?? 0) * 1000;
+ evictionMs = (evictionSeconds ?? 0) * 1000;
+ recordScanInterval = (scanIntervalSeconds ?? 0) * 1000;
+ }
+ if (!preserveLoadedConfiguration && !ttlFromLoad) {
+ if (keepApplicationTtl || (!declaredHere && !byNumber)) applicationTtl = 'all';
+ else if (declaredHere) applicationTtl = 'none';
+ else if (applicationTtl === 'none') applicationTtl = 'expiration';
}
- if (expirationMs < 0) throw new Error('Expiration can not be negative');
if (declaredHere && !preserveLoadedConfiguration) ttlDeclaredBySchema = true;
+ const scanWasScheduled = !preserveLoadedConfiguration && (expirationScanScheduled || cleanupTimer);
if (!preserveLoadedConfiguration) {
- // default to one quarter of the total expiration+eviction window
- recordCleanupInterval = recordScanInterval || (expirationMs + evictionMs) / 4;
- cleanupInterval = recordCleanupInterval;
- expirationScanScheduled = true;
+ expirationScanScheduled = Boolean(recordScanInterval || expirationMs || evictionMs);
+ cleanupInterval = recordScanInterval || (expirationMs + evictionMs) / 4 || DEFAULT_CLEANUP_INTERVAL;
+ if (scanWasScheduled && !expirationScanScheduled) retainExpiryScan = true;
}
if (
replicaTableKnown &&
@@ -3978,7 +4031,7 @@ export function makeTable(options): TableResourceClass {
} else if (replicaEvictionArmed) armReplicaEviction();
// Re-evaluate an existing table-level scan after an ownership-only declaration, but do not
// create the default daily cleanup timer for a table that has only an @expiresAt field.
- if (!preserveLoadedConfiguration || expirationScanScheduled || evictionMs) scheduleCleanup();
+ if (expirationScanScheduled || evictionMs || retainExpiryScan) scheduleCleanup();
// @expiresAt has its own interval rather than the cleanup timer above. Arm it whenever a live
// declaration introduces the attribute, including after this application already claimed TTL.
if (expiresAtProperty && !recordExpirationInterval) runRecordExpirationEviction();
@@ -7709,7 +7762,14 @@ export function makeTable(options): TableResourceClass {
// Evaluate at most once per table (on the first expiring write); later writes short-circuit on one check.
if (!expirationWarningChecked) {
expirationWarningChecked = true;
- if (!expirationMs && !evictionMs && !expirationScanScheduled && !expiresAtProperty && !hasSourceGet) {
+ if (
+ !expirationMs &&
+ !evictionMs &&
+ !expirationScanScheduled &&
+ !retainExpiryScan &&
+ !expiresAtProperty &&
+ !hasSourceGet
+ ) {
logger.warn?.(
`A per-record expiresAt was set on table "${tableName}" which has no table-level expiration/eviction, no expiresAt attribute, and no source; expiration will not be reliably enforced (the eviction scan is only armed best-effort on write and does not survive a restart with no writes). Configure a table-level expiration/eviction or an indexed expiresAt attribute for reliable eviction.`
);
@@ -12303,7 +12363,7 @@ export function makeTable(options): TableResourceClass {
if (disposed || !maintenanceClosed) return;
maintenanceClosed = false;
lastCleanupInterval = undefined;
- if (expirationScanScheduled || evictionMs) scheduleCleanup();
+ if (expirationScanScheduled || evictionMs || retainExpiryScan) scheduleCleanup();
if (replicaEvictionArmed) scheduleReplicaSweep();
if (expiresAtProperty && !recordExpirationInterval) runRecordExpirationEviction();
}
@@ -12390,14 +12450,15 @@ export function makeTable(options): TableResourceClass {
defaultAllowReads.add(TableResource.prototype.allowRead);
try {
TableResource.updatedAttributes(); // on creation, update accessors as well
- if (expirationMs || evictionMs) {
+ // anything but absent or zero is validated here: a peer's definition reaches a create unvalidated
+ if ((expirationMs !== undefined && expirationMs !== 0) || evictionMs !== 0) {
ttlFromLoad = true;
try {
- TableResource.setTTLExpiration(expirationMs / 1000);
+ TableResource.setTTLExpiration({ expiration: (expirationMs ?? 0) / 1000, eviction: evictionMs / 1000 });
} finally {
ttlFromLoad = false;
}
- }
+ } else if (retainExpiryScan) scheduleCleanup();
if (expiresAtProperty && !recordExpirationInterval) runRecordExpirationEviction();
} catch (error) {
TableResource.cleanup();
@@ -14424,11 +14485,6 @@ export function makeTable(options): TableResourceClass {
if (ownsCleanup()) {
// run on the last thread so we aren't overloading lower-numbered threads
if (cleanupTimer) clearTimeout(cleanupTimer);
- if (!cleanupInterval) {
- // no replacement pass is being scheduled, so nothing is left to settle a superseded one
- settlePendingCleanup();
- return;
- }
// This pass adopts the awaiters of the pass whose timer it just cleared: they settle when
// this pass's scan completes, so a reclamation run is never told the storage was reclaimed
// before any scan ran. It has to run now, though — that run blocks its whole path on the
@@ -14567,7 +14623,7 @@ export function makeTable(options): TableResourceClass {
function runRecordExpirationEviction() {
if (isReadOnlyMode()) return;
// Periodically evict expired records, searching for records who expiresAt timestamp is before now
- if (ownsStoreExpiration(primaryStore.path) || (ttlConfiguredByApplication && isDedicatedWorker())) {
+ if (ownsStoreExpiration(primaryStore.path) || (ttlOwnedByThread && isDedicatedWorker())) {
// we want to run the pruning of expired records on only one thread so we don't have conflicts in evicting
recordExpirationInterval = setInterval(() => {
// go through each database and table and then search for expired entries
@@ -15185,7 +15241,7 @@ export function makeTable(options): TableResourceClass {
));
}
function ownsCleanup(): boolean {
- return ownsStoreMaintenance(primaryStore.path) || (ttlConfiguredByApplication && isDedicatedWorker());
+ return ownsStoreMaintenance(primaryStore.path) || (ttlOwnedByThread && isDedicatedWorker());
}
// an already-armed thread may own the timer only now (the main thread under threads: 0 loads
// tables before it is worker 0), so the flag and the timer are settled on every wake
diff --git a/resources/databases.ts b/resources/databases.ts
index 466462801c..fdc303b0c3 100644
--- a/resources/databases.ts
+++ b/resources/databases.ts
@@ -44,6 +44,9 @@ import {
makeTable,
replicaExpirationMsOf,
replicaRefreshFractionOf,
+ describeTtlValue,
+ savedTtlSeconds,
+ ttlSecondsOf,
ignoreAlreadyDropped,
acquireUpdateAttributesLock,
releaseUpdateAttributesLock,
@@ -1137,6 +1140,30 @@ export function catalogCreatedTime(table: { dbisDB: any; tableName: string; prim
const createdTime = primaryCatalogRowFor(table)?.value?.createdTime;
return Number.isFinite(createdTime) ? createdTime : undefined;
}
+function savedTtlOf(row: any): { expiration?: number; eviction?: number } {
+ return { expiration: savedTtlSeconds(row.expiration), eviction: savedTtlSeconds(row.eviction) };
+}
+/** Whether a declared TTL field (undefined = not declared, 0 = cleared) differs from the catalog's. */
+function ttlChanged(declared: number | undefined, saved: unknown): boolean {
+ if (declared === undefined) return false;
+ return declared ? Number(saved) !== declared : saved !== undefined;
+}
+const warnedInvalidSavedTtl = new Set();
+/**
+ * A saved TTL field as a load applies it: a value no declaration could save now (a negative eviction or a
+ * non-numeric expiration an older create persisted) loads as absent, and the next declaration rewrites it.
+ */
+function loadedTtlSeconds(databaseName: string, tableName: string, field: string, saved: unknown) {
+ const seconds = savedTtlSeconds(saved);
+ const key = `${databaseName}.${tableName}.${field}`;
+ if (seconds === undefined && saved !== undefined && saved !== 0 && !warnedInvalidSavedTtl.has(key)) {
+ logger.warn(
+ `Ignoring the saved ${field} ${describeTtlValue(saved)} of ${databaseName}.${tableName}: not a valid TTL`
+ );
+ warnedInvalidSavedTtl.add(key);
+ }
+ return seconds;
+}
/**
* The table-level expiration and eviction (seconds) on disk, each present only when set. A class on a thread
* that application code configured can hold a TTL the catalog does not.
@@ -1148,14 +1175,25 @@ export function catalogExpiration(table: { dbisDB: any; tableName: string; prima
// the row declareTable writes the settings to and initStores loads them from: a named primary row wins
const named = table.primaryKey ? table.dbisDB?.getSync(`${table.tableName}/${table.primaryKey}`) : undefined;
const row = named?.isPrimaryKey ? named : table.dbisDB?.getSync(`${table.tableName}/`);
- // a create persists the declared value as given, which can be a numeric string
- const expiration = Number(row?.expiration);
- const eviction = Number(row?.eviction);
+ // a create persisted the declared value as given, which can be a numeric string
+ const expiration = savedTtlSeconds(row?.expiration);
+ const eviction = savedTtlSeconds(row?.eviction);
const settings: { expiration?: number; eviction?: number } = {};
- if (expiration > 0 && Number.isFinite(expiration)) settings.expiration = expiration;
- if (eviction > 0 && Number.isFinite(eviction)) settings.eviction = eviction;
+ if (expiration) settings.expiration = expiration;
+ if (eviction) settings.eviction = eviction;
return settings;
}
+/**
+ * Arms the cleanup scan of every loaded table that has a TTL. Under `threads: 0` the main thread loads every
+ * table before it becomes worker 0, when no thread can own a scan yet.
+ */
+export function armTableCleanupScans(): void {
+ const loaded = getDatabases();
+ // system is not enumerable
+ for (const database of new Set([loaded.system, ...Object.values(loaded)])) {
+ for (const Table of Object.values(database ?? {})) (Table as any)?._armCleanupScan?.();
+ }
+}
/**
* Backfills the stamp of a generation created on a build that stored none, once a peer's stamped definition
* proves which generation it is. Only an absent stamp is written; false when one exists or the table is gone.
@@ -2158,8 +2196,8 @@ function initStores(
const audit =
typeof primaryAttribute.audit === 'boolean' ? primaryAttribute.audit : envGet(CONFIG_PARAMS.LOGGING_AUDITLOG);
const trackDeletes = primaryAttribute.trackDeletes;
- const expiration = primaryAttribute.expiration;
- const eviction = primaryAttribute.eviction;
+ const expiration = loadedTtlSeconds(databaseName, tableName, 'expiration', primaryAttribute.expiration);
+ const eviction = loadedTtlSeconds(databaseName, tableName, 'eviction', primaryAttribute.eviction);
const replicaExpiration = primaryAttribute.replicaExpiration;
const replicaRefreshFraction = primaryAttribute.replicaRefreshFraction;
const sealed = primaryAttribute.sealed;
@@ -2338,7 +2376,7 @@ function initStores(
if (!destination) requestComputedIndexRepair(table);
if (replicaExpiration !== undefined || replicaRefreshFraction !== undefined)
table.setTTLExpiration({ replicaExpiration, replicaRefreshFraction, fromSchema: true });
- table._adoptCatalogExpiration(expiration, eviction);
+ table._adoptCatalogExpiration(expiration, eviction, primaryAttribute.expiryScan === true);
} else {
if (recreateForTableIdChange) table.cleanup();
table = setTable(
@@ -2358,6 +2396,7 @@ function initStores(
replicaExpirationMS: replicaExpirationMsOf(replicaExpiration),
replicaRefreshFraction: replicaRefreshFractionOf(replicaRefreshFraction),
replicaDeadlines: Boolean(tableDef.replicaDeadlines),
+ expiryScan: primaryAttribute.expiryScan === true,
cacheControl,
trackDeletes,
tableName,
@@ -2905,9 +2944,10 @@ interface TableDefinition {
path?: string;
/** Kept when a peer's definition creates the table here, so every copy of a generation shares one stamp. */
createdTime?: number;
- expiration?: number;
- eviction?: number;
- scanInterval?: number;
+ /** Seconds; 0 or null clears a saved value, undefined (a caller not authoring the schema) keeps it. */
+ expiration?: number | null;
+ eviction?: number | null;
+ scanInterval?: number | null;
/** RocksDB-only replica lifetime in seconds; 0 disables admission. LMDB always proxies. */
replicaExpiration?: number;
/** RocksDB-only renewal fraction in (0, 1]; 1 never renews. */
@@ -4360,18 +4400,27 @@ function declareTable(target: TableTarget, tableDefinition: T
if (isBranchIdentity(databaseName)) {
throw new ClientError(`'${databaseName}' is in use as a branch store identity and cannot be a database name`);
}
+ // Before anything is opened, created or persisted. A peer's values reach only a table this call creates, as
+ // a catalog load does (see the end of this function).
+ if (origin !== 'cluster') {
+ const qualifiedName = `${databaseName}.${tableName}`;
+ expiration = ttlSecondsOf(qualifiedName, 'expiration', expiration);
+ eviction = ttlSecondsOf(qualifiedName, 'eviction', eviction);
+ scanInterval = ttlSecondsOf(qualifiedName, 'scanInterval', scanInterval);
+ }
const rootStore = target.rootStore(databaseName, tableName);
const tables = target.tables(databaseName);
logger.trace(`Defining ${tableName} in ${databaseName}`);
let Table = tables?.[tableName];
+ const namesOneTtlField = origin !== 'cluster' && (expiration === undefined) !== (eviction === undefined);
+ // the saved value of the field such a declaration omits, from the row persistTableSettings writes
+ let inheritedTtl: { expiration?: number; eviction?: number } | undefined;
if (rootStore.status === 'closed') {
throw new Error(`Can not use a closed data store for ${tableName}`);
}
- // A re-declaration persists these to the catalog, where a negative expiration fails every later load, and
- // nothing it changes before then (index builds included) is rolled back.
+ // A re-declaration persists these to the catalog, and nothing it changes before then (index builds included)
+ // is rolled back.
if (Table && origin !== 'cluster') {
- if (+expiration < 0 || +eviction < 0)
- throw new ClientError(`Expiration and eviction of '${databaseName}.${tableName}' can not be negative`);
if (replicaExpiration != null) replicaExpirationMsOf(replicaExpiration);
if (replicaRefreshFraction != null) replicaRefreshFractionOf(replicaRefreshFraction);
}
@@ -5786,10 +5835,15 @@ function declareTable(target: TableTarget, tableDefinition: T
// A peer's expiration reaches a table this call created through makeTable, as a catalog load. It never
// configures an existing table, and applying it here would mark the TTL as this thread's own declaration.
if (origin === 'cluster') expiration = eviction = scanInterval = undefined;
+ if (inheritedTtl) {
+ expiration ??= inheritedTtl.expiration ?? 0;
+ eviction ??= inheritedTtl.eviction ?? 0;
+ }
if (
expiration ||
eviction ||
scanInterval ||
+ ((expiration === 0 || eviction === 0) && (Table.expirationMS || Table.evictionMS)) ||
replicaExpiration !== undefined ||
replicaRefreshFraction !== undefined ||
attributes.some((attribute) => attribute.expiresAt)
@@ -5819,6 +5873,7 @@ function declareTable(target: TableTarget, tableDefinition: T
}
function persistTableSettings(attribute: any, dbiKey: string) {
const attributeDescriptor = attributesDbi.getSync(dbiKey) || {};
+ if (namesOneTtlField) inheritedTtl = savedTtlOf(attributeDescriptor);
// Persist schemaDefined when the explicit live value disagrees with disk. Without this,
// a stale `false` (from a v4-era write or replicated event) survives every reload: the
// in-memory re-assert in the existing-Table branch only fixes the worker that ran @table,
@@ -5833,8 +5888,8 @@ function declareTable(target: TableTarget, tableDefinition: T
(typeof audit === 'boolean' && audit !== attributeDescriptor.audit) ||
(sealed !== undefined && sealed !== attributeDescriptor.sealed) ||
(replicate !== undefined && replicate !== attributeDescriptor.replicate) ||
- (expiration && +expiration !== +attributeDescriptor.expiration) ||
- (eviction && +eviction !== +attributeDescriptor.eviction) ||
+ ttlChanged(expiration, attributeDescriptor.expiration) ||
+ ttlChanged(eviction, attributeDescriptor.eviction) ||
(replicaExpiration != null && replicaExpiration !== attributeDescriptor.replicaExpiration) ||
(replicaRefreshFraction != null && replicaRefreshFraction !== attributeDescriptor.replicaRefreshFraction) ||
attribute.type !== attributeDescriptor.type)
@@ -5842,13 +5897,23 @@ function declareTable(target: TableTarget, tableDefinition: T
exclusiveLock();
const currentPrimaryAttribute = attributesDbi.getSync(dbiKey);
if (!currentPrimaryAttribute || tableIsDropping(currentPrimaryAttribute, dbiKey)) return;
+ if (namesOneTtlField) inheritedTtl = savedTtlOf(currentPrimaryAttribute);
const updatedPrimaryAttribute = { ...currentPrimaryAttribute };
if (typeof audit === 'boolean') {
if (audit) Table.enableAuditing();
updatedPrimaryAttribute.audit = audit;
}
- if (expiration) updatedPrimaryAttribute.expiration = +expiration;
- if (eviction) updatedPrimaryAttribute.eviction = +eviction;
+ // Rows already stamped under a TTL being cleared keep their expiry, and only a cleanup scan removes
+ // them: a load of a table marked this way keeps the daily scan a table without a TTL would not arm.
+ if (
+ (expiration === 0 && savedTtlSeconds(currentPrimaryAttribute.expiration)) ||
+ (eviction === 0 && savedTtlSeconds(currentPrimaryAttribute.eviction))
+ )
+ updatedPrimaryAttribute.expiryScan = true;
+ if (expiration) updatedPrimaryAttribute.expiration = expiration;
+ else if (expiration === 0) delete updatedPrimaryAttribute.expiration;
+ if (eviction) updatedPrimaryAttribute.eviction = eviction;
+ else if (eviction === 0) delete updatedPrimaryAttribute.eviction;
if (replicaExpiration != null) {
replicaExpirationMsOf(replicaExpiration);
updatedPrimaryAttribute.replicaExpiration = replicaExpiration;
diff --git a/resources/defineTable.ts b/resources/defineTable.ts
index 2b882fa40c..726b577000 100644
--- a/resources/defineTable.ts
+++ b/resources/defineTable.ts
@@ -401,6 +401,8 @@ function compileTypeDef(name: string, shape: Shape, options: DefineTableOptions)
properties,
schemaDefined: true,
...tableOptions,
+ expiration: tableOptions.expiration ?? null,
+ eviction: tableOptions.eviction ?? null,
};
}
diff --git a/resources/graphql.ts b/resources/graphql.ts
index c077d48ff2..3cdc66d914 100644
--- a/resources/graphql.ts
+++ b/resources/graphql.ts
@@ -288,7 +288,9 @@ async function processGraphQLSchema(
// Boolean directive args arrive as actual booleans; tolerate string forms too.
if (typeDef.randomAccessFields !== undefined)
typeDef.randomAccessFields = typeDef.randomAccessFields === true || typeDef.randomAccessFields === 'true';
- // schema is authoritative for cacheControl: null (vs undefined) clears a removed directive on reload;
+ typeDef.expiration ??= null;
+ typeDef.eviction ??= null;
+ // schema is authoritative for the TTL and cacheControl: null (vs undefined) clears a removed directive on reload;
// coerce to a header-safe string so a non-string arg or CRLF can't reach the HTTP response
typeDef.cacheControl =
typeDef.cacheControl == null
diff --git a/server/threads/socketRouter.ts b/server/threads/socketRouter.ts
index 989b7e801e..ef4f399a65 100644
--- a/server/threads/socketRouter.ts
+++ b/server/threads/socketRouter.ts
@@ -21,6 +21,7 @@ import { startTransactionLogCooling } from '../transactionLogCooling.ts';
import { applyConfiguredV8Flags } from './v8Flags.ts';
import { startLongLivedTransactionReporting } from '../../resources/longLivedTransactions.ts';
import { initializeFullTextQueryPolicy } from '../../resources/indexes/fullTextNativeBinding.ts';
+import { armTableCleanupScans } from '../../resources/databases.ts';
import {
effectiveHttpThreadCount,
isWorkerPoolActive,
@@ -108,6 +109,10 @@ export async function startHTTPThreads(threadCount = 2, dynamicThreads?: boolean
if (isProcessShuttingDown()) return [];
await threadServer.startServers();
if (isProcessShuttingDown()) return [];
+ // After the components' declarations, so a stale saved TTL never arms a scan that a schema replaces.
+ // Workers need no such pass: they have a worker index before loading components, so each table
+ // arms its own scan as it loads.
+ armTableCleanupScans();
// startServers() schedules listener startup after loading components; await its cached
// batch so a bind failure reaches bin/run.ts and exits non-zero in single-thread mode too.
await threadServer.listenOnPorts();
diff --git a/unitTests/resources/createTableCatalogOrder.test.js b/unitTests/resources/createTableCatalogOrder.test.js
index f686292927..594fc32aaf 100644
--- a/unitTests/resources/createTableCatalogOrder.test.js
+++ b/unitTests/resources/createTableCatalogOrder.test.js
@@ -191,7 +191,11 @@ describe('create table catalog write order', () => {
'a create that fails before makeTable must not register the class'
);
assert.strictEqual(releasedReclamationHandlers.length, 0, 'nothing was registered before makeTable');
- assert.throws(() => table({ ...definition(), expiration: -1 }), /Expiration can not be negative/);
+ // a peer's definition is not validated at the declaration boundary, so its bad value fails in makeTable
+ assert.throws(
+ () => table({ ...definition(), origin: 'cluster', expiration: -1 }),
+ /must be a non-negative number of seconds/
+ );
assert.strictEqual(
databases.test[tableName],
undefined,
diff --git a/unitTests/resources/tableExpirationPersistence.test.js b/unitTests/resources/tableExpirationPersistence.test.js
index 2c6916d541..335882aed9 100644
--- a/unitTests/resources/tableExpirationPersistence.test.js
+++ b/unitTests/resources/tableExpirationPersistence.test.js
@@ -10,8 +10,10 @@ const {
resetDatabases,
table,
} = require('#src/resources/databases');
+const { defineTable, types } = require('#src/resources/defineTable');
+const { loadGQLSchema } = require('#src/resources/graphql');
-const DATABASE = 'test';
+const DATABASE = 'ttl_persistence';
async function commitCatalog(Table) {
if (Table.dbisDB.committed) await Table.dbisDB.committed;
@@ -222,6 +224,22 @@ describe('table-level expiration persistence (harper-pro#1179)', () => {
assert.strictEqual(Created.expirationMS, 3600000);
});
+ it("rejects a peer definition's invalid TTL on a table it creates, naming the field", function () {
+ for (const [suffix, settings, field] of [
+ ['Eviction', { expiration: 60, eviction: -1 }, 'eviction'],
+ ['EvictionOnly', { eviction: -1 }, 'eviction'],
+ ['NonNumeric', { expiration: 'soon' }, 'expiration'],
+ ]) {
+ const tableName = `ExpiryPeerInvalid${suffix}`;
+ assert.throws(
+ () => keyed(tableName, { origin: 'cluster', schemaDefined: undefined, ...settings }),
+ new RegExp(`^Error: ${field} of '${DATABASE}.${tableName}' must be a non-negative number`),
+ suffix
+ );
+ assert.strictEqual(databases[DATABASE][tableName], undefined, suffix);
+ }
+ });
+
it("leaves an existing table's settings alone when a peer definition differs", async function () {
const tableName = 'ExpiryPeerExisting';
const Local = keyed(tableName, { expiration: 600 });
@@ -245,4 +263,246 @@ describe('table-level expiration persistence (harper-pro#1179)', () => {
assert.strictEqual(Local.evictionMS, 0);
assert.deepStrictEqual(catalogExpiration(Local), { expiration: 600 });
});
+ it('clears a saved expiration and eviction that a re-declaration sets to 0, null or "0" (#3934)', async function () {
+ for (const [suffix, cleared] of [
+ ['Zero', { expiration: 0, eviction: 0 }],
+ ['Null', { expiration: null, eviction: null }],
+ ['String', { expiration: '0', eviction: '0' }],
+ ]) {
+ const tableName = `ExpiryCleared${suffix}`;
+ const Declared = keyed(tableName, { expiration: 600, eviction: 300 });
+ keyed(tableName, cleared);
+ await commitCatalog(Declared);
+ const { value } = primaryRow(Declared);
+ assert(!('expiration' in value) && !('eviction' in value), `${suffix}: ${JSON.stringify(value)}`);
+ assert.deepStrictEqual(catalogExpiration(Declared), {}, suffix);
+ assert(!Declared.expirationMS, `${suffix}: this thread still applies ${Declared.expirationMS}ms`);
+ assert.strictEqual(Declared.evictionMS, 0, suffix);
+
+ delete databases[DATABASE][tableName];
+ resetDatabases();
+ const Restarted = getDatabases()[DATABASE][tableName];
+ assert(!Restarted.expirationMS, `${suffix}: a restart loads ${Restarted.expirationMS}ms`);
+ assert.strictEqual(Restarted.evictionMS, 0, suffix);
+ }
+ });
+
+ it('clears only the setting a re-declaration sets to 0', async function () {
+ const tableName = 'ExpiryClearedEvictionOnly';
+ const Declared = keyed(tableName, { expiration: 600, eviction: 300 });
+ keyed(tableName, { expiration: 600, eviction: 0 });
+ await commitCatalog(Declared);
+ assert.deepStrictEqual(catalogExpiration(Declared), { expiration: 600 });
+ assert.strictEqual(Declared.evictionMS, 0);
+ });
+
+ it('drops a TTL this thread loaded from the catalog when its re-declaration clears it', async function () {
+ const tableName = 'ExpiryLoadedThenCleared';
+ const Loaded = keyed(tableName);
+ await writeCatalogSettings(Loaded, { expiration: 120, eviction: 60 });
+ resetDatabases();
+ assert.strictEqual(Loaded.expirationMS, 120000);
+
+ keyed(tableName, { expiration: 0 });
+ await commitCatalog(Loaded);
+ assert(!Loaded.expirationMS, `this thread still applies ${Loaded.expirationMS}ms`);
+ assert.deepStrictEqual(catalogExpiration(Loaded), { eviction: 60 }, 'an omitted eviction stays saved');
+ assert.strictEqual(Loaded.evictionMS, 60000, 'and stays applied');
+ });
+
+ it('keeps the saved value of the setting a re-declaration omits, on disk and in memory', async function () {
+ const tableName = 'ExpiryPartialUpdate';
+ const Declared = keyed(tableName, { expiration: 60, eviction: 3600 });
+ keyed(tableName, { expiration: 120 });
+ await commitCatalog(Declared);
+ assert.deepStrictEqual(catalogExpiration(Declared), { expiration: 120, eviction: 3600 });
+ assert.strictEqual(Declared.expirationMS, 120000);
+ assert.strictEqual(Declared.evictionMS, 3600000);
+ });
+
+ it('keeps the omitted setting as the row it writes holds it, not as an earlier read saw it', async function () {
+ const tableName = 'ExpiryInheritedUnderLock';
+ const Declared = keyed(tableName, { expiration: 60, eviction: 300 });
+ await commitCatalog(Declared);
+ const { key } = primaryRow(Declared);
+ const catalog = Declared.dbisDB;
+ const originalGetSync = catalog.getSync;
+ let raced = false;
+ catalog.getSync = function (readKey, ...rest) {
+ const value = originalGetSync.call(this, readKey, ...rest);
+ if (readKey === key && !raced) {
+ raced = true;
+ // another thread saves a new eviction right after this declaration's first read of the row
+ catalog.putSync(key, { ...value, eviction: 3600 });
+ }
+ return value;
+ };
+ try {
+ keyed(tableName, { expiration: 120 });
+ } finally {
+ catalog.getSync = originalGetSync;
+ }
+ await commitCatalog(Declared);
+ assert.deepStrictEqual(catalogExpiration(Declared), { expiration: 120, eviction: 3600 });
+ assert.strictEqual(Declared.evictionMS, 3600000);
+ });
+
+ it('does not clear a TTL application code configured when the schema declares none', async function () {
+ const tableName = 'ExpiryApplicationKept';
+ const Configured = keyed(tableName);
+ Configured.setTTLExpiration({ expiration: 30, eviction: 10 });
+ keyed(tableName, { expiration: null, eviction: null });
+ assert.strictEqual(Configured.expirationMS, 30000);
+ assert.strictEqual(Configured.evictionMS, 10000);
+ });
+
+ it('keeps a TTL application code set through a re-declaration with none, isolated or not (QA-5424)', async function () {
+ for (const isolatedApplicationOwner of [true, false]) {
+ const tableName = `ExpiryApplicationRedeclared${isolatedApplicationOwner ? 'Isolated' : 'Shared'}`;
+ const Configured = keyed(tableName, { expiration: null, eviction: null, isolatedApplicationOwner });
+ Configured.setTTLExpiration({ expiration: 30, eviction: 10 });
+ keyed(tableName, { expiration: null, eviction: null, isolatedApplicationOwner });
+ assert.strictEqual(Configured.expirationMS, 30000, `isolatedApplicationOwner: ${isolatedApplicationOwner}`);
+ assert.strictEqual(Configured.evictionMS, 10000, `isolatedApplicationOwner: ${isolatedApplicationOwner}`);
+ await commitCatalog(Configured);
+ assert.deepStrictEqual(catalogExpiration(Configured), {}, 'an application TTL is never saved');
+ }
+ });
+
+ it('keeps only the expiration a number-form call set over a schema TTL the schema then drops', async function () {
+ const tableName = 'ExpiryNumberFormOverSchema';
+ const Declared = keyed(tableName, { expiration: 60, eviction: 3600, scanInterval: 3600 });
+ Declared.setTTLExpiration(30);
+ keyed(tableName, { expiration: null, eviction: null });
+ assert.strictEqual(Declared.expirationMS, 30000);
+ assert.strictEqual(Declared.evictionMS, 0, "the eviction was the schema's");
+ keyed(tableName, { expiration: null, eviction: null });
+ assert.strictEqual(Declared.expirationMS, 30000, 'a further declaration without a TTL keeps it');
+ });
+
+ it("clears a TTL an isolated application's schema declared when it declares none", async function () {
+ const tableName = 'ExpiryIsolatedSchemaCleared';
+ const Declared = keyed(tableName, { expiration: 600, eviction: 300, isolatedApplicationOwner: true });
+ assert.strictEqual(Declared.expirationMS, 600000);
+ keyed(tableName, { expiration: null, eviction: null, isolatedApplicationOwner: true });
+ await commitCatalog(Declared);
+ assert.deepStrictEqual(catalogExpiration(Declared), {});
+ assert(!Declared.expirationMS, `this thread still applies ${Declared.expirationMS}ms`);
+ assert.strictEqual(Declared.evictionMS, 0);
+ });
+
+ it('clears a schema TTL that replaced the one application code set, when the schema drops it', async function () {
+ for (const isolatedApplicationOwner of [true, false]) {
+ const tableName = `ExpirySchemaReplacedApplication${isolatedApplicationOwner ? 'Isolated' : 'Shared'}`;
+ const Configured = keyed(tableName, { isolatedApplicationOwner });
+ Configured.setTTLExpiration({ expiration: 30 });
+ keyed(tableName, { expiration: 120, isolatedApplicationOwner });
+ assert.strictEqual(Configured.expirationMS, 120000);
+ keyed(tableName, { expiration: null, eviction: null, isolatedApplicationOwner });
+ assert(
+ !Configured.expirationMS,
+ `isolatedApplicationOwner ${isolatedApplicationOwner} kept ${Configured.expirationMS}ms`
+ );
+ }
+ });
+
+ it('loads a saved value no declaration could save now as absent', async function () {
+ const tableName = 'ExpiryLegacyInvalid';
+ const Legacy = keyed(tableName);
+ await writeCatalogSettings(Legacy, { expiration: { legacy: 1n }, eviction: -1 });
+ delete databases[DATABASE][tableName];
+ resetDatabases();
+ const Restarted = getDatabases()[DATABASE][tableName];
+ assert(Restarted, 'the table must still load');
+ assert(!Restarted.expirationMS);
+ assert.strictEqual(Restarted.evictionMS, 0);
+ keyed(tableName, { expiration: 0, eviction: 0 });
+ await commitCatalog(Restarted);
+ const { value } = primaryRow(Restarted);
+ assert(!('expiration' in value) && !('eviction' in value), 'the next declaration rewrites the row');
+ });
+
+ it('brings a loaded class up to date when another thread clears the catalog (#3934)', async function () {
+ const tableName = 'ExpiryAdoptedClear';
+ const Loaded = keyed(tableName);
+ await writeCatalogSettings(Loaded, { expiration: 120, eviction: 60 });
+ resetDatabases();
+ assert.strictEqual(Loaded.expirationMS, 120000);
+
+ const { key, value } = primaryRow(Loaded);
+ const cleared = { ...value };
+ delete cleared.expiration;
+ delete cleared.eviction;
+ Loaded.dbisDB.putSync(key, cleared);
+ await commitCatalog(Loaded);
+ resetDatabases();
+ assert(!Loaded.expirationMS, `the reload kept ${Loaded.expirationMS}ms`);
+ assert.strictEqual(Loaded.evictionMS, 0);
+ });
+
+ it('keeps a saved TTL through callers that do not author it', async function () {
+ const tableName = 'ExpiryAddAttribute';
+ const Declared = keyed(tableName, { expiration: 600, eviction: 300 });
+ await Declared.addAttributes([{ name: 'added', type: 'String' }]);
+ await commitCatalog(Declared);
+ assert.deepStrictEqual(catalogExpiration(Declared), { expiration: 600, eviction: 300 });
+ assert.strictEqual(Declared.expirationMS, 600000);
+ });
+
+ it('clears a TTL that a @table directive no longer declares (#3934)', async function () {
+ const tableName = 'ExpiryGraphqlRemoved';
+ await loadGQLSchema(`type ${tableName} @table(database: "${DATABASE}", expiration: 600, eviction: 300) {
+ id: ID @primaryKey
+ }`);
+ const Declared = databases[DATABASE][tableName];
+ await commitCatalog(Declared);
+ assert.deepStrictEqual(catalogExpiration(Declared), { expiration: 600, eviction: 300 });
+
+ await loadGQLSchema(`type ${tableName} @table(database: "${DATABASE}") {
+ id: ID @primaryKey
+ }`);
+ await commitCatalog(Declared);
+ assert.deepStrictEqual(catalogExpiration(Declared), {});
+ assert(!Declared.expirationMS, `this thread still applies ${Declared.expirationMS}ms`);
+ assert.strictEqual(Declared.evictionMS, 0);
+ });
+
+ it('clears a TTL that defineTable options no longer declare (#3934)', async function () {
+ const tableName = 'ExpiryDefineTableRemoved';
+ const shape = { id: types.id.primaryKey };
+ const Declared = defineTable(tableName, shape, { database: DATABASE, expiration: 600 });
+ await commitCatalog(Declared);
+ assert.deepStrictEqual(catalogExpiration(Declared), { expiration: 600 });
+
+ defineTable(tableName, shape, { database: DATABASE });
+ await commitCatalog(Declared);
+ assert.deepStrictEqual(catalogExpiration(Declared), {});
+ assert(!Declared.expirationMS, `this thread still applies ${Declared.expirationMS}ms`);
+ });
+
+ it('rejects an invalid TTL before a create, naming the table and field (#1286, #3935)', async function () {
+ for (const [field, value] of [
+ ['expiration', -1],
+ ['eviction', -1],
+ ['scanInterval', -1],
+ ['expiration', 'soon'],
+ ['eviction', Infinity],
+ ['expiration', Number.NaN],
+ ['expiration', 1e308],
+ ['expiration', ''],
+ ['expiration', true],
+ ['eviction', [5]],
+ ]) {
+ const tableName = `ExpiryInvalidCreate_${field}_${String(value).replace(/\W/g, '')}`;
+ assert.throws(
+ () => keyed(tableName, { [field]: value }),
+ (error) =>
+ error.statusCode === 400 &&
+ error.message.includes(field) &&
+ error.message.includes(`${DATABASE}.${tableName}`),
+ `${field}: ${value}`
+ );
+ assert.strictEqual(databases[DATABASE][tableName], undefined, `${field}: ${value} created the table`);
+ }
+ });
});
diff --git a/unitTests/resources/ttlCleanupInterval.test.js b/unitTests/resources/ttlCleanupInterval.test.js
index f224328d86..5ccc99bf62 100644
--- a/unitTests/resources/ttlCleanupInterval.test.js
+++ b/unitTests/resources/ttlCleanupInterval.test.js
@@ -1,12 +1,46 @@
require('../testUtils');
const assert = require('assert');
const { setupTestDBPath } = require('../testUtils');
-const { table } = require('#src/resources/databases');
+const {
+ table,
+ databases,
+ getDatabases,
+ resetDatabases,
+ armTableCleanupScans,
+ scopedTableFactory,
+} = require('#src/resources/databases');
const { makeTable } = require('#src/resources/Table');
const { setMainIsWorker } = require('#src/server/threads/manageThreads');
const { waitFor } = require('../waitFor');
const { setTimeout: sleep } = require('node:timers/promises');
+async function timersArmedBy(run, settleMs = 0) {
+ const originalSetTimeout = global.setTimeout;
+ const delays = [];
+ global.setTimeout = (callback, delay, ...args) => {
+ delays.push(delay);
+ return originalSetTimeout(callback, delay, ...args);
+ };
+ try {
+ run();
+ if (settleMs) await sleep(settleMs);
+ } finally {
+ global.setTimeout = originalSetTimeout;
+ }
+ return delays;
+}
+// the TTLs these tests clear scan every 15 s or less
+const isDailyCadence = (delay) => (delay > 15_000 && delay <= 86_400_000) || isNextDailySlot(delay);
+// a first daily scan with nothing pending lands on the next whole day since the start of the year
+function isNextDailySlot(delay) {
+ const startOfYear = new Date();
+ startOfYear.setMonth(0, 1);
+ startOfYear.setHours(0, 0, 0); // keeps the milliseconds, as scheduleCleanup does
+ const now = Date.now();
+ const slot = Math.ceil((now - startOfYear.getTime()) / 86_400_000) * 86_400_000 + startOfYear.getTime();
+ return Math.abs(delay - (slot - now)) < 5000;
+}
+
// The cleanup scan is what removes an expired record: reads return it until then. Its interval has to come
// from the table's TTL whichever way the TTL was set. A worker that learns of a table from the catalog (a
// table created at runtime on another thread, or any table after a restart) arms it with the number form.
@@ -100,6 +134,31 @@ describe('TTL cleanup scan interval', () => {
}
});
+ it("an isolated application's schema without a TTL leaves the scan for one application code set (QA-5424)", async () => {
+ const definition = {
+ table: 'TtlIsolatedApplicationKept',
+ database: databaseName,
+ expiration: null,
+ eviction: null,
+ attributes: [{ name: 'id', isPrimaryKey: true }],
+ };
+ // the table as a previous start left it; application code (jsResource) then loads before the schema
+ const Table = table({ table: definition.table, database: databaseName, attributes: definition.attributes });
+ const isolatedTable = scopedTableFactory(undefined, true);
+ try {
+ Table.sourcedFrom({ get: (id) => ({ id }) }, { expiration: 0.5 });
+ isolatedTable(definition);
+ isolatedTable(definition);
+ assert.strictEqual(Table.expirationMS, 500);
+ await Table.put(1, { id: 1 });
+ await Table.primaryStore.committed;
+ assert(stored(Table, 1));
+ await waitFor(() => !stored(Table, 1), { timeout: 5000, message: 'the expired record was never evicted' });
+ } finally {
+ Table.cleanup();
+ }
+ });
+
it('the number form keeps a configured scanInterval', () => {
const Table = table({
table: 'TtlScanInterval',
@@ -154,4 +213,229 @@ describe('TTL cleanup scan interval', () => {
Hydrated.cleanup();
}
});
+ it('an eviction-only TTL adopted from the catalog arms the scan (#3935)', async () => {
+ const Declared = table({
+ table: 'TtlAdoptedEvictionOnly',
+ database: databaseName,
+ attributes: [{ name: 'id', isPrimaryKey: true }],
+ });
+ const Hydrated = hydrate(Declared, 'TtlAdoptedEvictionOnly', {});
+ try {
+ const delays = await timersArmedBy(() => Hydrated._adoptCatalogExpiration(0, 0.05));
+ assert.strictEqual(Hydrated.expirationMS, 0);
+ assert.strictEqual(Hydrated.evictionMS, 50);
+ assert.strictEqual(delays.length, 1, 'adopting an eviction-only TTL must arm the scan');
+ await Hydrated.put(1, { id: 1 }, { expiresAt: Date.now() });
+ await Hydrated.primaryStore.committed;
+ await waitFor(() => !stored(Hydrated, 1), { timeout: 5000, message: 'the evictable record was never evicted' });
+ } finally {
+ Hydrated.cleanup();
+ }
+ });
+
+ it('adopting a cleared TTL leaves the scan at the daily default (#3934)', async () => {
+ const Declared = table({
+ table: 'TtlAdoptedClear',
+ database: databaseName,
+ attributes: [{ name: 'id', isPrimaryKey: true }],
+ });
+ const Hydrated = hydrate(Declared, 'TtlAdoptedClear', { expirationMS: 60_000 });
+ try {
+ const delays = await timersArmedBy(() => Hydrated._adoptCatalogExpiration(undefined, undefined), 200);
+ assert.strictEqual(Hydrated.expirationMS, 0);
+ assert.strictEqual(Hydrated.evictionMS, 0);
+ assert(delays.some(isDailyCadence), `no daily scan armed: ${delays}`);
+ } finally {
+ Hydrated.cleanup();
+ }
+ });
+
+ it('clearing a TTL with zeros keeps a scan for per-record expiry instead of disabling it', async () => {
+ const Table = table({
+ table: 'TtlZeroed',
+ database: databaseName,
+ attributes: [{ name: 'id', isPrimaryKey: true }],
+ });
+ try {
+ Table.setTTLExpiration({ expiration: 60 });
+ const delays = await timersArmedBy(() => Table.setTTLExpiration({ expiration: 0, eviction: 0 }), 200);
+ assert(delays.some(isDailyCadence), `the scan was disabled instead of moved to the daily default: ${delays}`);
+ } finally {
+ Table.cleanup();
+ }
+ });
+
+ it('rejects a negative or non-finite value without applying any of it (#3935)', () => {
+ const Table = table({
+ table: 'TtlRejected',
+ database: databaseName,
+ attributes: [{ name: 'id', isPrimaryKey: true }],
+ });
+ try {
+ Table.setTTLExpiration({ expiration: 10, eviction: 5 });
+ for (const opts of [
+ { expiration: 1, eviction: -1 },
+ { expiration: -1, eviction: 1 },
+ { expiration: 1, scanInterval: -1 },
+ { expiration: Infinity },
+ { expiration: Number.NaN },
+ -1,
+ ]) {
+ assert.throws(
+ () => Table.setTTLExpiration(opts),
+ /must be a non-negative number of seconds/,
+ JSON.stringify(opts)
+ );
+ assert.strictEqual(Table.expirationMS, 10_000, `${JSON.stringify(opts)} changed the expiration`);
+ assert.strictEqual(Table.evictionMS, 5_000, `${JSON.stringify(opts)} changed the eviction`);
+ }
+ } finally {
+ Table.cleanup();
+ }
+ });
+
+ it('a table loaded before the main thread became a worker arms its scan once it is one (#2831)', async () => {
+ const tableName = 'TtlLoadedBeforePromotion';
+ const definition = { table: tableName, database: databaseName, attributes: [{ name: 'id', isPrimaryKey: true }] };
+ const Declared = table(definition);
+ // a restart: the row outlives this class, whose scan stops with it; the TTL comes after the commit, so
+ // no scan of this class can reach the row
+ const expiresAt = Date.now() + 1000;
+ await Declared.put(1, { id: 1 }, { expiresAt });
+ await Declared.primaryStore.committed;
+ table({ ...definition, expiration: 0.05 });
+ Declared.cleanup();
+ delete databases[databaseName][tableName];
+ let Loaded;
+ try {
+ // threads: 0 loads every table on the main thread before setMainIsWorker(true)
+ setMainIsWorker(false);
+ resetDatabases();
+ Loaded = getDatabases()[databaseName][tableName];
+ } finally {
+ setMainIsWorker(true);
+ }
+ try {
+ assert.strictEqual(Loaded.expirationMS, 50);
+ await sleep(Math.max(0, expiresAt + 300 - Date.now()));
+ assert(stored(Loaded, 1), 'a scan ran although no thread owned it at load');
+ armTableCleanupScans();
+ await waitFor(() => !stored(Loaded, 1), { timeout: 5000, message: 'the expired record was never evicted' });
+ } finally {
+ Loaded.cleanup();
+ }
+ });
+
+ it('a TTL cleared once the main thread is a worker still scans the rows it stamped (#2831, #3934)', async () => {
+ const Declared = table({
+ table: 'TtlClearedAfterPromotion',
+ database: databaseName,
+ attributes: [{ name: 'id', isPrimaryKey: true }],
+ });
+ let Hydrated;
+ try {
+ // threads: 0 loads the table before the main thread owns any scan, then the schema clears its TTL
+ setMainIsWorker(false);
+ Hydrated = hydrate(Declared, 'TtlClearedAfterPromotion', { expirationMS: 60_000 });
+ } finally {
+ setMainIsWorker(true);
+ }
+ try {
+ const delays = await timersArmedBy(
+ () => Hydrated.setTTLExpiration({ expiration: 0, eviction: 0, fromSchema: true }),
+ 200
+ );
+ assert(delays.some(isDailyCadence), `no scan armed for the rows the cleared TTL stamped: ${delays}`);
+ } finally {
+ Hydrated.cleanup();
+ }
+ });
+
+ it('keeps a scan for rows a cleared TTL stamped after a restart (#3934)', async () => {
+ const tableName = 'TtlClearedThenRestarted';
+ const attributes = () => [{ name: 'id', isPrimaryKey: true }];
+ const Declared = table({ table: tableName, database: databaseName, expiration: 60, attributes: attributes() });
+ table({ table: tableName, database: databaseName, expiration: 0, attributes: attributes() });
+ if (Declared.dbisDB.committed) await Declared.dbisDB.committed;
+ const primaryRow = Declared.dbisDB.getSync(`${tableName}/`) ?? Declared.dbisDB.getSync(`${tableName}/id`);
+ assert.strictEqual(primaryRow.expiryScan, true, JSON.stringify(primaryRow));
+ Declared.cleanup();
+ delete databases[databaseName][tableName];
+ let Restarted;
+ try {
+ // loaded with no scan owner, so the only timer below is the one this table arms
+ setMainIsWorker(false);
+ resetDatabases();
+ Restarted = getDatabases()[databaseName][tableName];
+ } finally {
+ setMainIsWorker(true);
+ }
+ try {
+ assert(!Restarted.expirationMS);
+ const delays = await timersArmedBy(() => Restarted._armCleanupScan());
+ assert(delays.length === 1 && isNextDailySlot(delays[0]), `no daily scan armed: ${delays}`);
+ } finally {
+ Restarted.cleanup();
+ }
+ });
+
+ it('a loaded class arms the scan when a reload finds the catalog marked expiryScan', async () => {
+ const Declared = table({
+ table: 'TtlReloadExpiryScan',
+ database: databaseName,
+ attributes: [{ name: 'id', isPrimaryKey: true }],
+ });
+ const Hydrated = hydrate(Declared, 'TtlReloadExpiryScan', {});
+ try {
+ const delays = await timersArmedBy(() => Hydrated._adoptCatalogExpiration(undefined, undefined, true));
+ assert(delays.length === 1 && isNextDailySlot(delays[0]), `no daily scan armed: ${delays}`);
+ } finally {
+ Hydrated.cleanup();
+ }
+ });
+
+ it('keeps a scan for rows a cleared TTL stamped when maintenance resumes (#3934)', async () => {
+ const Table = table({
+ table: 'TtlClearedThenResumed',
+ database: databaseName,
+ attributes: [{ name: 'id', isPrimaryKey: true }],
+ });
+ try {
+ Table.setTTLExpiration({ expiration: 60 });
+ Table.setTTLExpiration({ expiration: 0, eviction: 0 });
+ await Table.closeMaintenance();
+ const delays = await timersArmedBy(() => Table.resumeMaintenance());
+ assert(delays.some(isDailyCadence), `resuming maintenance armed no scan: ${delays}`);
+ } finally {
+ Table.cleanup();
+ }
+ });
+
+ it('arms no scan for a table with no TTL', async () => {
+ const Table = table({
+ table: 'TtlNoneToArm',
+ database: databaseName,
+ attributes: [{ name: 'id', isPrimaryKey: true }],
+ });
+ try {
+ assert.deepStrictEqual(await timersArmedBy(() => Table._armCleanupScan()), []);
+ } finally {
+ Table.cleanup();
+ }
+ });
+
+ it('arms system tables too, which databases does not enumerate', () => {
+ const systemTables = Object.values(databases.system);
+ assert(systemTables.length > 0);
+ assert(!Object.keys(databases).includes('system'), 'the system database is enumerable now; simplify this test');
+ const armed = new Set();
+ const originals = systemTables.map((Table) => [Table, Table._armCleanupScan]);
+ for (const Table of systemTables) Table._armCleanupScan = () => armed.add(Table);
+ try {
+ armTableCleanupScans();
+ } finally {
+ for (const [Table, original] of originals) Table._armCleanupScan = original;
+ }
+ assert.strictEqual(armed.size, systemTables.length);
+ });
});