diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml index eea54fce2a..9c93ac973b 100644 --- a/.github/workflows/unit-test.yml +++ b/.github/workflows/unit-test.yml @@ -330,7 +330,7 @@ jobs: run: npm run build - name: Verify the published native binding loads - run: node --input-type=module --eval "const binding = await import('@harperfast/fulltext/native'); if (typeof binding.openNativeFullTextIndex !== 'function' || typeof binding.openNativeFullTextReader !== 'function') throw new Error('@harperfast/fulltext loaded without native writer and reader openers');" + run: node --input-type=module --eval "const binding = await import('@harperfast/fulltext/native'); if (typeof binding.openNativeFullTextIndex !== 'function' || typeof binding.openNativeFullTextReader !== 'function' || typeof binding.NativeFullTextIndex?.prototype.searchPage !== 'function' || (await binding.runtimeInfo()).queryApiVersion !== 4) throw new Error('@harperfast/fulltext loaded without the required writer, reader, and query API 4 contract');" - name: Native full-text lifecycle test - run: npx mocha unitTests/resources/nativeFullTextDerivedIndexLifecycle.test.js unitTests/resources/nativeFullTextTableSearch.test.js + run: npx mocha unitTests/resources/nativeFullTextDerivedIndexLifecycle.test.js unitTests/resources/nativeFullTextTableSearch.test.js --forbid-pending --fail-zero diff --git a/config-root.schema.json b/config-root.schema.json index 59cb1b8328..b77d6b9280 100644 --- a/config-root.schema.json +++ b/config-root.schema.json @@ -580,6 +580,75 @@ "description": "Storage engine configuration.", "additionalProperties": true, "properties": { + "fullText": { + "type": "object", + "description": "Full-text request work limits and native process budgets. Native process budgets require a Harper process restart to change; they do not cap total process memory or disk usage.", + "additionalProperties": false, + "properties": { + "requestBudgetMilliseconds": { + "type": "integer", + "minimum": 1, + "maximum": 9007199254740991, + "default": 5000 + }, + "maxRawHitsPerRequest": { + "type": "integer", + "minimum": 1, + "maximum": 9007199254740991, + "default": 10000, + "description": "Maximum fetched native hits per full-text request, including bounded prefix replay." + }, + "maxResultsPerRequest": { + "type": "integer", + "minimum": 1, + "maximum": 9007199254740991, + "default": 10000, + "description": "Maximum offset plus limit retained by a full-text request, independent of fetched-hit work." + }, + "runtime": { + "type": "object", + "additionalProperties": false, + "properties": { + "maxResidentIndexes": { + "type": "integer", + "minimum": 1, + "maximum": 4294967295, + "default": 128 + }, + "maxIndexingThreads": { + "type": "integer", + "minimum": 1, + "maximum": 4294967295, + "default": 128 + }, + "maxSearchThreads": { + "type": "integer", + "minimum": 1, + "maximum": 4294967295, + "default": 256 + }, + "maxWriterMemoryBytes": { + "type": "integer", + "minimum": 1, + "maximum": 9007199254740991, + "default": 4294967296 + }, + "maxQueuedBytes": { + "type": "integer", + "minimum": 1, + "maximum": 9007199254740991, + "default": 8589934592 + }, + "maxExpensiveSearches": { + "type": "integer", + "minimum": 1, + "maximum": 4294967295, + "default": 8 + } + } + } + } + }, "writeAsync": { "type": "boolean", "description": "When true, sets LMDB noSync to disable LMDB's commit disk sync for faster writes (risk of losing recent commits on OS crash or power loss); when false, LMDB keeps commit disk sync enabled (overlapped on Linux and macOS by default). Applies only to the LMDB storage engine; this setting has no effect on RocksDB, where Harper does not fsync individual commits to disk (transaction logs are synced on memtable flush rather than per commit). Default: false" diff --git a/integrationTests/resources/full-text-search.test.ts b/integrationTests/resources/full-text-search.test.ts index f697b6193e..62648f511f 100644 --- a/integrationTests/resources/full-text-search.test.ts +++ b/integrationTests/resources/full-text-search.test.ts @@ -148,7 +148,7 @@ suite('deployed full-text fields and native search', (ctx: ContextWithHarper) => const packageJson = JSON.parse(await readFile(resolve(import.meta.dirname, '../..', 'package.json'), 'utf8')); assert.strictEqual(info.packageVersion, packageJson.optionalDependencies['@harperfast/fulltext']); assert.strictEqual(info.mutationBatchApiVersion, 5); - assert.strictEqual(info.queryApiVersion, 3); + assert.strictEqual(info.queryApiVersion, 4); assert.ok(info.queryClassIsolationMinimumSearchThreads > 0); const description = await operation({ operation: 'describe_table', schema: 'data', table: 'Product' }); const index = description.full_text_indexes.find(({ name }) => name === 'catalogSearch'); @@ -284,6 +284,43 @@ suite('deployed full-text fields and native search', (ctx: ContextWithHarper) => ); }); + test('rejects removed paging functions before a collection delete can run', async () => { + for (const option of ['continuation()', 'continuation(token)', 'closeContinuation()']) { + for (const method of ['GET', 'DELETE']) { + const response = await fetch(`${ctx.harper.httpURL}/Product/?catalogSearch=matches=pagination&${option}`, { + method, + headers: { Authorization: authorization }, + signal: AbortSignal.timeout(45000), + }); + assert.strictEqual(response.ok, false); + assert.strictEqual((await response.json()).code, 'SyntaxViolation'); + } + } + assert.deepStrictEqual(ids(await search(query('pagination'))), PAGE_IDS); + }); + + test('fills REST autocomplete pages after structured filtering and beyond the first native batch', async () => { + const all = await search(query('pagination')); + const expected = all + .filter(({ category }) => category === 'even') + .slice(0, 100) + .map(({ id }) => id); + for (const comparator of ['matches_prefix', 'matches_fuzzy_prefix']) { + const results = await request( + `/Product/?catalogSearch=${comparator}=pagin&category=even&limit(100)&select(id,title)` + ); + assert.deepStrictEqual( + results.map(({ id }) => id), + expected + ); + const larger = await request(`/Product/?catalogSearch=${comparator}=pagin&limit(150)&select(id,title)`); + assert.deepStrictEqual( + larger.map(({ id }) => id), + all.slice(0, 150).map(({ id }) => id) + ); + } + }); + test('preserves source permissions and row filters for non-admin searches', async () => { await operation({ operation: 'add_role', @@ -578,4 +615,92 @@ suite('deployed full-text fields and native search', (ctx: ContextWithHarper) => await waitForReady('Product'); await waitForIds(query('waterproof'), ['one', 'two']); }); + test( + 'applies full-text request limits through set_configuration and worker restart', + { skip: process.platform === 'win32' ? 'HTTP worker restart on Windows: HarperFast/harper#549' : false }, + async () => { + async function restartWorkers() { + const { job_id } = await operation({ operation: 'restart_service', service: 'http_workers' }); + assert(job_id); + const deadline = Date.now() + 60000; + let status; + do { + let job; + try { + [job] = await operation({ operation: 'get_job', id: job_id }); + } catch (error) { + const code = (error as any)?.cause?.code ?? (error as any)?.code; + if (!['ECONNREFUSED', 'ECONNRESET', 'UND_ERR_SOCKET'].includes(code)) throw error; + assert.strictEqual( + ctx.harper.process.exitCode, + null, + `Harper exited during worker restart: ${ctx.harper.process.exitCode}` + ); + assert.strictEqual( + ctx.harper.process.signalCode, + null, + `Harper stopped during worker restart: ${ctx.harper.process.signalCode}` + ); + } + status = job?.status; + if (status === 'COMPLETE' || status === 'ERROR') break; + await delay(25); + } while (Date.now() < deadline); + assert.strictEqual(status, 'COMPLETE'); + await waitForReady('Product'); + } + async function assertBudget(name: string) { + let status: number; + let body: string; + const deadline = Date.now() + 10000; + do { + const response = await fetch(`${ctx.harper.httpURL}/SearchProduct/`, { + method: 'POST', + headers: { 'Authorization': authorization, 'Content-Type': 'application/json' }, + body: JSON.stringify(query('shoe', {}, { limit: 2 })), + signal: AbortSignal.timeout(5000), + }); + status = response.status; + body = await response.text(); + if (status === 400 && body.includes(name)) break; + await delay(25); + } while (Date.now() < deadline); + assert.strictEqual(status, 400, body); + assert.ok(body.includes(name), body); + } + const original = (await operation({ operation: 'get_configuration' })).storage.fullText; + assert.ok(original, 'Installed configuration must include full-text defaults'); + try { + await operation({ + operation: 'set_configuration', + storage_fullText_maxRawHitsPerRequest: 500, + storage_fullText_maxResultsPerRequest: 1, + storage_fullText_requestBudgetMilliseconds: 1000, + }); + const updated = (await operation({ operation: 'get_configuration' })).storage.fullText; + assert.strictEqual(updated.maxRawHitsPerRequest, 500); + assert.strictEqual(updated.maxResultsPerRequest, 1); + assert.strictEqual(updated.requestBudgetMilliseconds, 1000); + await restartWorkers(); + await assertBudget('maxResultsPerRequest'); + await operation({ + operation: 'set_configuration', + storage_fullText_maxRawHitsPerRequest: 1, + storage_fullText_maxResultsPerRequest: 500, + }); + await restartWorkers(); + await assertBudget('maxRawHitsPerRequest'); + } finally { + if (ctx.harper.process.exitCode === null && ctx.harper.process.signalCode === null) { + await operation({ + operation: 'set_configuration', + storage_fullText_maxRawHitsPerRequest: original.maxRawHitsPerRequest, + storage_fullText_maxResultsPerRequest: original.maxResultsPerRequest, + storage_fullText_requestBudgetMilliseconds: original.requestBudgetMilliseconds, + }); + await restartWorkers(); + } + } + } + ); }); diff --git a/package-lock.json b/package-lock.json index 494980457d..74ae160d12 100644 --- a/package-lock.json +++ b/package-lock.json @@ -145,7 +145,7 @@ "node": "^22.18.0 || >=24.0.0" }, "optionalDependencies": { - "@harperfast/fulltext": "0.5.0", + "@harperfast/fulltext": "0.6.0", "@harperfast/hnsw": "0.4.0", "bufferutil": "4.1.0", "segfault-handler": "1.3.0" @@ -2409,25 +2409,25 @@ "license": "Apache-2.0" }, "node_modules/@harperfast/fulltext": { - "version": "0.5.0", - "resolved": "https://registry.npmjs.org/@harperfast/fulltext/-/fulltext-0.5.0.tgz", - "integrity": "sha512-rH2Q3LJLHPhxi8muzPLg00STbO7RFIqAgLA3tpBzSRdsJVuCV9ZMO2StlSZnUAPMnF5wF9SieuhHT9wi1vINBQ==", + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/@harperfast/fulltext/-/fulltext-0.6.0.tgz", + "integrity": "sha512-j5PO3L6j0mTSFLtGSaiNSYS07JCekehNdDhoyYitazvcoVUWKOgOeGQ6N3fxwUPIKW/tdWI8aYaRE0g4J0exaQ==", "license": "Apache-2.0", "optional": true, "engines": { "node": "^22.18.0 || >=24.0.0" }, "optionalDependencies": { - "@harperfast/fulltext-darwin-arm64": "0.5.0", - "@harperfast/fulltext-linux-arm64-gnu": "0.5.0", - "@harperfast/fulltext-linux-x64-gnu": "0.5.0", - "@harperfast/fulltext-win32-x64-msvc": "0.5.0" + "@harperfast/fulltext-darwin-arm64": "0.6.0", + "@harperfast/fulltext-linux-arm64-gnu": "0.6.0", + "@harperfast/fulltext-linux-x64-gnu": "0.6.0", + "@harperfast/fulltext-win32-x64-msvc": "0.6.0" } }, "node_modules/@harperfast/fulltext-darwin-arm64": { - "version": "0.5.0", - "resolved": "https://registry.npmjs.org/@harperfast/fulltext-darwin-arm64/-/fulltext-darwin-arm64-0.5.0.tgz", - "integrity": "sha512-0OtKonopmXaP0/9wvpCuo/sklNxOEiQWnVf7aUsthH0B3sKUYT7HGnGtYkzqqUxdsFNlHPBr3IShyrWx9nAWYQ==", + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/@harperfast/fulltext-darwin-arm64/-/fulltext-darwin-arm64-0.6.0.tgz", + "integrity": "sha512-GJMxSRgiZK7fYXH7ftGMve35SssJuc2d15ypyL2ZfUppREs85jdF6BDCx9lQfsX7y2MdBvuyHYz0fFRjiBRycQ==", "cpu": [ "arm64" ], @@ -2441,9 +2441,9 @@ } }, "node_modules/@harperfast/fulltext-linux-arm64-gnu": { - "version": "0.5.0", - "resolved": "https://registry.npmjs.org/@harperfast/fulltext-linux-arm64-gnu/-/fulltext-linux-arm64-gnu-0.5.0.tgz", - "integrity": "sha512-QfNlYUntkal56jZejQ8cbSj8qvUlYv21V2j9xVctkFv+gmfn892kdU4oCg7dR7Qift8/huKUOyL/eGO0qYbgGw==", + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/@harperfast/fulltext-linux-arm64-gnu/-/fulltext-linux-arm64-gnu-0.6.0.tgz", + "integrity": "sha512-RntyUDmN+hivxjKPkEcjvu6cb5dLMrv43jEydJPJqzcTM9ywMei/tuWRXCqk3U5iZ3FvX65wpk4CTlA816L71g==", "cpu": [ "arm64" ], @@ -2460,9 +2460,9 @@ } }, "node_modules/@harperfast/fulltext-linux-x64-gnu": { - "version": "0.5.0", - "resolved": "https://registry.npmjs.org/@harperfast/fulltext-linux-x64-gnu/-/fulltext-linux-x64-gnu-0.5.0.tgz", - "integrity": "sha512-+g5LMXhtODen+pGgmyTxcCuUAlk2NTeFOvr6duqUBVx542Hd/uDqPC9tUMjL12LJ9ci3LFqGwgW3FkxGedT7KA==", + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/@harperfast/fulltext-linux-x64-gnu/-/fulltext-linux-x64-gnu-0.6.0.tgz", + "integrity": "sha512-4BtUwQJoMomHCjv7Bet7nwx6Hs0Oq4u7hys7KorbI6pgFi1nHb6wN2xbf4//iIr8IhyLZUWpxlo6hbU9ojUqTw==", "cpu": [ "x64" ], @@ -2479,9 +2479,9 @@ } }, "node_modules/@harperfast/fulltext-win32-x64-msvc": { - "version": "0.5.0", - "resolved": "https://registry.npmjs.org/@harperfast/fulltext-win32-x64-msvc/-/fulltext-win32-x64-msvc-0.5.0.tgz", - "integrity": "sha512-PseSz47BMcIOMQdYUFoAUqAEyeHzceX+QQFun0wVCu/PtERPvtPWzzZxV4ofySzz+c9FkJHTSDu4XbAZhkaqkA==", + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/@harperfast/fulltext-win32-x64-msvc/-/fulltext-win32-x64-msvc-0.6.0.tgz", + "integrity": "sha512-5eLVRRoDjJuwC3199Ij1APH3HvIBMANJQHQLDeFa1oDm5ORHL+403WcrnzBV9kVZm2XpjDberfrzqPrQK0fVFw==", "cpu": [ "x64" ], @@ -8266,6 +8266,102 @@ "license": "Apache-2.0", "peer": true }, + "node_modules/harper/node_modules/@harperfast/fulltext": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/@harperfast/fulltext/-/fulltext-0.5.0.tgz", + "integrity": "sha512-rH2Q3LJLHPhxi8muzPLg00STbO7RFIqAgLA3tpBzSRdsJVuCV9ZMO2StlSZnUAPMnF5wF9SieuhHT9wi1vINBQ==", + "dev": true, + "license": "Apache-2.0", + "optional": true, + "peer": true, + "engines": { + "node": "^22.18.0 || >=24.0.0" + }, + "optionalDependencies": { + "@harperfast/fulltext-darwin-arm64": "0.5.0", + "@harperfast/fulltext-linux-arm64-gnu": "0.5.0", + "@harperfast/fulltext-linux-x64-gnu": "0.5.0", + "@harperfast/fulltext-win32-x64-msvc": "0.5.0" + } + }, + "node_modules/harper/node_modules/@harperfast/fulltext-darwin-arm64": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/@harperfast/fulltext-darwin-arm64/-/fulltext-darwin-arm64-0.5.0.tgz", + "integrity": "sha512-0OtKonopmXaP0/9wvpCuo/sklNxOEiQWnVf7aUsthH0B3sKUYT7HGnGtYkzqqUxdsFNlHPBr3IShyrWx9nAWYQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "peer": true, + "engines": { + "node": "^22.18.0 || >=24.0.0" + } + }, + "node_modules/harper/node_modules/@harperfast/fulltext-linux-arm64-gnu": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/@harperfast/fulltext-linux-arm64-gnu/-/fulltext-linux-arm64-gnu-0.5.0.tgz", + "integrity": "sha512-QfNlYUntkal56jZejQ8cbSj8qvUlYv21V2j9xVctkFv+gmfn892kdU4oCg7dR7Qift8/huKUOyL/eGO0qYbgGw==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": "^22.18.0 || >=24.0.0" + } + }, + "node_modules/harper/node_modules/@harperfast/fulltext-linux-x64-gnu": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/@harperfast/fulltext-linux-x64-gnu/-/fulltext-linux-x64-gnu-0.5.0.tgz", + "integrity": "sha512-+g5LMXhtODen+pGgmyTxcCuUAlk2NTeFOvr6duqUBVx542Hd/uDqPC9tUMjL12LJ9ci3LFqGwgW3FkxGedT7KA==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": "^22.18.0 || >=24.0.0" + } + }, + "node_modules/harper/node_modules/@harperfast/fulltext-win32-x64-msvc": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/@harperfast/fulltext-win32-x64-msvc/-/fulltext-win32-x64-msvc-0.5.0.tgz", + "integrity": "sha512-PseSz47BMcIOMQdYUFoAUqAEyeHzceX+QQFun0wVCu/PtERPvtPWzzZxV4ofySzz+c9FkJHTSDu4XbAZhkaqkA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ], + "peer": true, + "engines": { + "node": "^22.18.0 || >=24.0.0" + } + }, "node_modules/harper/node_modules/@harperfast/rocksdb-js": { "version": "2.10.0", "resolved": "https://registry.npmjs.org/@harperfast/rocksdb-js/-/rocksdb-js-2.10.0.tgz", diff --git a/package.json b/package.json index 44a98f8a66..2041a3bd48 100644 --- a/package.json +++ b/package.json @@ -287,7 +287,7 @@ } }, "optionalDependencies": { - "@harperfast/fulltext": "0.5.0", + "@harperfast/fulltext": "0.6.0", "@harperfast/hnsw": "0.4.0", "bufferutil": "4.1.0", "segfault-handler": "1.3.0" diff --git a/resources/DESIGN.md b/resources/DESIGN.md index 82dddee83b..fb80798f59 100644 --- a/resources/DESIGN.md +++ b/resources/DESIGN.md @@ -41,6 +41,7 @@ User-facing syntax: [Full-text search](FULL_TEXT.md). - Compile `name: FullText @fullText(...)` into `fullTextIndexes` and the internal durable `fullTextFields` name set. Programmatic declarations default that name set to every index name. Keep write/projection guards independent of query-reader readiness; the name set does not distinguish earlier declarations from new ones. Never add the declaration to `attributes` or record `properties`: it has no stored, computed, or selectable value. Source fields own permissions. - Accept only field placement with its name supplied by the field. Reject a `name` argument, table-level declarations, and index names that collide with current stored attributes. Earlier beta declarations and indexes have no supported upgrade path. - Hold one reader lease for the entire search, including every native result page and highlight trace. Reload only an idle reader. Publication during an active query opens a replacement for subsequent queries and retires the old reader after its lease drains. Offset paging across different native snapshots can duplicate or skip hits; deduplication cannot recover skipped hits. +- Native process budgets are captured before `startHTTPThreads()` loads application components, or on an earlier main-thread native load/managed spawn. Managed worker data and worker environment data carry the snapshot; standalone application workers created before capture must fail before configuring the addon. Validate the raw snapshot at activation; worker replacement cannot reconfigure a process-global addon. Non-runtime query policies still resolve live. - Native fulltext owns commits and reader snapshots. Harper owns audit coverage, worker epochs, readiness, and publication notifications. A stable native snapshot does not imply a global record snapshot during concurrent writes; source versions are checked when hydrating hits. Sources: [schema parser](graphql.ts), [declaration compiler and generation identity](fullTextSchema.ts), [table read/write guards](Table.ts), [reader leases and paging](indexes/fullTextQueryIndex.ts), [publication-between-pages regression](../unitTests/resources/fullTextQueryIndex.test.js). diff --git a/resources/FULL_TEXT.md b/resources/FULL_TEXT.md index 216a4cba03..dba61e1ff4 100644 --- a/resources/FULL_TEXT.md +++ b/resources/FULL_TEXT.md @@ -29,7 +29,7 @@ The supported analyzer is `english@2`. `stopWords`, `positions`, and `surfaceTer From an authenticated Resource handler, request table permission checks and pass its context through to `Product.search()`: ```javascript -return Product.search( +return tables.Product.search( { checkPermission: true, conditions: [ @@ -88,14 +88,67 @@ Weight and highlighting changes preserve the physical index and refresh query re ## Native package and limits -Harper pins `@harperfast/fulltext` to **0.5.0** and validates the native runtime capabilities before activation. Full-text activation requires RocksDB and an audited table. Declare new schema tables with `@table(audit: true)` and keep audit logging enabled while an index is declared. LMDB is rejected. A missing or incompatible native package prevents activation. +Harper pins `@harperfast/fulltext` to **0.6.0** and validates the native runtime capabilities before activation. Full-text activation requires RocksDB and an audited table. Declare new schema tables with `@table(audit: true)` and keep audit logging enabled while an index is declared. LMDB is rejected. A missing or incompatible native package prevents activation. A format-incompatible native package upgrade causes each node to rebuild its local derived index from authoritative records. Full-text queries on that node return 503 until the index is ready. Use a rolling upgrade to keep full-text search available on other replicas while each local index rebuilds. Earlier beta declarations and indexes are unsupported. This field-only API does not provide a compatibility or upgrade path for them. -Native queries have finite result windows and execution budgets. Reduce the requested offset/limit or narrow the query when a window is exceeded; filtering can also exhaust the window. Prefix modes use the native autocomplete window, clamped when necessary so its maximum result set fits in one native response. Highlight tracing has separate record and source-byte bounds. These are functional constraints, not catalog-scale performance guarantees. +## Paging and work limits -Harper does not configure Fulltext's native filter metadata or submit native filter or candidate-ID requests. Ordinary conditions continue through Harper's secondary-index gates and authoritative record predicates. +Full-text search uses Harper's usual `limit` and `offset` options. `Table.search()` returns an iterable, including when a custom Resource forwards or filters it. -For implementation details, see the [full-text invariants in the Resource design guide](DESIGN.md#full-text-declarations-and-reader-snapshots). The schema contract lives in [schema.graphql](../schema.graphql); the compiler and generation rules are in [fullTextSchema.ts](fullTextSchema.ts). +```javascript +return tables.Product.search( + { + checkPermission: true, + conditions: [{ attribute: 'catalogSearch', comparator: 'matches', value: 'trail shoes' }], + offset: 20, + limit: 20, + }, + this.getContext() +); +``` + +The equivalent REST range has an inclusive start and exclusive end: + +```text +GET /Product/?catalogSearch=matches=trail%20shoes&limit(20,40) +``` + +Each page runs a new query. Changes to records, permissions, or the index can move results between requests, so separate pages may omit or repeat records. There is no retained search or cross-request snapshot. + +Within one request, Harper reads native batches on the same index reader until enough eligible records have been found or the matches are exhausted. Current source versions and record filters are checked before counting records toward the offset and limit. This traversal also lets prefix and fuzzy-prefix searches return more than the initial autocomplete batch. The library's ordinary search window is not a total-result cutoff for a bounded Harper request. + +Use a bounded `limit` for large result sets. An unbounded query that exceeds the native result window returns 400. Offset requests revisit earlier matches; deep offsets and restrictive filters can exhaust the work or time budget. Harper can retain up to `offset + limit` accepted records while building a bounded result, so raising the work limit can also increase memory use. An exhausted search returns its remaining eligible records, including an empty result. A budget failure returns an error instead of a successful incomplete page. + +Search settings are under `storage.fullText`: + +| Setting | Default | Meaning | +| --------------------------- | ------: | ----------------------------------------------------------------------------------------------------------------- | +| `maxRawHitsPerRequest` | 10,000 | Fetched native hits, including bounded prefix replay | +| `maxResultsPerRequest` | 10,000 | Maximum retained offset plus limit, independent of fetched-hit work | +| `requestBudgetMilliseconds` | 5,000 | Execution budget across native batches, source checks and highlight tracing, capped by the native execution limit | + +The hit limit counts fetched hits, not every record Tantivy visits while scoring. Reaching it before the page fills or the search is exhausted returns 400. Exceeding the execution budget returns 503. Narrow the query, reduce the offset, or increase the relevant configured budget. Freshness waiting uses `waitForIndexMilliseconds` separately. Reader acquisition, query construction and some native operations can take additional time; the execution budget is not a hard response-time guarantee. + +Each setting has the usual configuration parameter name, such as `storage_fullText_maxRawHitsPerRequest`, for `set_configuration` and environment overrides. After changing request limits, restart HTTP workers with `restart_service` (`service: "http_workers"`) or restart Harper. Native process-budget changes require a full process restart. + +Native process admission settings live under `storage.fullText.runtime`: + +| Setting | Default | +| ---------------------- | ------: | +| `maxResidentIndexes` | 128 | +| `maxIndexingThreads` | 128 | +| `maxSearchThreads` | 256 | +| `maxWriterMemoryBytes` | 4 GiB | +| `maxQueuedBytes` | 8 GiB | +| `maxExpensiveSearches` | 8 | + +These limits apply together. For example, two indexing threads and 64 MiB of writer memory per index reach the default thread and memory budgets at 64 indexes. Readers share an index's runtime; they do not allocate another writer. Admission limits are not hard bounds on total resident memory or disk usage. + +Harper captures native runtime budgets before loading application components, or on an earlier main-thread native load or managed worker spawn. Application-created workers inherit that configuration. A worker created before capture fails with `FullTextWorkerInitializationError` (500, non-retryable); recreate it after Harper initializes. Replacement workers keep the established budgets after a live configuration change. Restart the Harper process to apply new native runtime budgets; restarting only HTTP workers is insufficient. + +Harper does not configure the library's native filter metadata or submit native filter or candidate-ID requests. Ordinary conditions continue through Harper's secondary-index gates and authoritative record predicates. Native page, response-byte and highlight limits still apply. Fulltext 0.6.0 also limits each analyzed query leaf to 64 terms and the compiled query to 256 clauses. Raising Harper's work budgets does not raise those limits. See the [published library limits](https://github.com/HarperFast/fulltext/blob/v0.6.0/docs/limits.md) for the full list and recovery guidance. Configuration of those query limits is tracked in [Make query term and clause limits configurable with bounded resource policy](https://github.com/HarperFast/fulltext/issues/63). + +For implementation details, see the [full-text invariants in the Resource design guide](DESIGN.md#full-text-declarations-and-reader-snapshots). The schema contract lives in [schema.graphql](../schema.graphql); compiler and generation rules live in [fullTextSchema.ts](fullTextSchema.ts). diff --git a/resources/indexes/DESIGN.md b/resources/indexes/DESIGN.md index bf3ba566dd..f41077b206 100644 --- a/resources/indexes/DESIGN.md +++ b/resources/indexes/DESIGN.md @@ -527,13 +527,21 @@ single record larger than the native trace limit is rejected before crossing the Search hits carry the source record version. Harper reads the authoritative record in its transaction and omits a hit when the version no longer matches; it never combines an old score or highlight with new content. Bounded searches begin by over-fetching 32–256 native hits. When version checks or structured filters under-fill a page, Harper sizes the next request from observed yield. Pages grow within the native response envelope, with 4,096 as an absolute ceiling, then shrink as selectivity improves; a zero-yield page grows geometrically because it provides no selectivity estimate. -Each page yields after at most 256 estimated source and companion-index point reads to check cancellation and the execution deadline. Equality point probes are limited to four index reads per candidate; wider equality plans use the authoritative record predicate. The native result window remains the hard bound, and one reader lease keeps every page on one native snapshot. +Each page yields after at most 256 estimated source and companion-index point reads to check cancellation and the execution deadline. Equality point probes are limited to four index reads per candidate; wider equality plans use the authoritative record predicate. The requested visible `offset + limit` is bounded independently by `maxResultsPerRequest` and the raw-hit budget, not the native ordinary result window. Raising raw work for restrictive filters does not raise the retained-record target. One reader lease keeps every native batch in a request on the same snapshot. Harper does not configure native `filterFields` or send native `filter` or `candidateIds` payloads. Candidate planning, companion-index probes and the authoritative record predicate remain Harper-owned; the wrapper's candidate limits do not cap those mechanisms. -A lower-bound total must describe retrievable hits: while more hits remain, a page must return its requested limit. Fulltext 0.5.0 checks its deadline around the complete Tantivy collection and returns `E_TIMEOUT` instead of a partial page. Harper fails closed on a short page claiming more hits. Stale-version exhaustion observed in loaded records or a bounded sample of companion-index rejections is retryable index lag; filter-only exhaustion asks the client to narrow the query. Stale rejections outside that sample can still be classified as filter exhaustion; the bounded diagnostic avoids unbounded source reads. During bounded lag, results can temporarily omit a recent change when native hits end before the window. +Keep the first batch on `reader.search` so a filled bounded ordinary query stays in the native ordinary-search lane. Fulltext 0.6.0 classifies every traversal call as expensive (`protocol::search_is_expensive`), sharing the process permit budget and expensive per-index queue with deep/fuzzy work. A native-reader microbenchmark alone misses that admission/isolation difference. Unbounded exact-total queries already use expensive admission. The released classifier is [protocol.rs:449–472](https://github.com/HarperFast/fulltext/blob/v0.6.0/src/protocol.rs#L449-L472); [native.rs:477–478](https://github.com/HarperFast/fulltext/blob/v0.6.0/src/native.rs#L477-L478) selects its queue. -Harper clamps autocomplete to the maximum-size hits that fit one native response without reducing ordinary search. The response-capacity calculation is pinned to the Fulltext 0.5.0 Query API v3 framing: 13 response bytes plus 13 bytes per versioned hit before string data. A later query API must advertise or version its framing overhead before Harper accepts it. REST exposes the index coverage header; callers requiring current coverage use `maxIndexLagMilliseconds: 0` or `waitForIndexMilliseconds`. Full-text score descending is the only supported ordering in this release. Count requests return `recordCount: null` and `recordCountExact: false`; the native candidate total cannot become an exact Harper count after authorization, structured filtering and source-version checks. +When more hits are needed, switch to `searchPage` on the same reader. Ordinary search supplies no continuation, so validate and skip its bounded prefix before processing new hits. Native order is descending score then ascending native document-ID UTF-8 bytes; disagreement, repeated boundaries or empty pages claiming continuations fail closed. The real-addon regression verifies the collectors agree across modes and page sizes. + +Byte-aware batch sizes adapt to filter yield. Oversized responses retry the unchanged boundary at a smaller size under the same deadline. Exhausting the snapshot returns remaining eligible rows. `maxRawHitsPerRequest` caps fetched hits including bounded replay; `maxResultsPerRequest` independently caps retained offset plus limit. Both default to 10,000. Before replay, the first page total supplies a lower bound on the new hits needed to fill or exhaust the request; reject immediately if even that work plus replay cannot fit. Use the smaller of the remaining target and remaining native total so a shorter exhausted result remains valid. A lower-bound total equal to the first batch does not prove that additional hits exist, so only replay can be reserved; traversal must distinguish actual exhaustion from more matches before rejecting. Hitting a bound before exhaustion or filling the requested page throws an explicit error. These are not bounds on Tantivy scorer visits or uninterrupted query construction. Unbounded queries that reach a result/work ceiling must use a limit; they never silently return the retained prefix. + +Harper requires Fulltext 0.6.0, native ABI 9 and Query API 4. Traversal response capacity reserves 82 bytes before hits plus 13 bytes per versioned hit before ID/version strings; advertised page and response limits determine the batch ceiling. + +Read request policy from its three registered leaf parameters and runtime policy from `storage_fullText_runtime`. The legacy flattener does not retain camel-case aggregate containers such as `storage_fullText`; using that aggregate silently selects defaults. HTTP request settings follow normal worker-restart semantics; native runtime settings remain captured for the full process. + +REST exposes the index coverage header; callers requiring current coverage use `maxIndexLagMilliseconds: 0` or `waitForIndexMilliseconds`. Full-text score descending is the only supported ordering. Exact count requests return `recordCount: null` and `recordCountExact: false` for full-text custom-index traversal; estimated counts may use the planner unless an opaque row/vector filter suppresses them. Native totals cannot become exact Harper counts after authorization, structured filtering and source-version checks. Each request releases its reader lease in `finally`. Separate limit/offset requests execute independently and may see different publications; Harper retains no search state between requests. A declared Blob source is part of one index document. An oversized, invalid UTF-8 or otherwise permanently unusable Blob makes that whole document unindexable rather than publishing a partial diff --git a/resources/indexes/fullTextNativeBinding.ts b/resources/indexes/fullTextNativeBinding.ts index e707225e11..2ae99370e4 100644 --- a/resources/indexes/fullTextNativeBinding.ts +++ b/resources/indexes/fullTextNativeBinding.ts @@ -1,11 +1,25 @@ import type { FullTextDerivedIndexEngine } from './fullTextDerivedIndex.ts'; +import * as env from '../../utility/environment/environmentManager.ts'; +import { CONFIG_PARAMS } from '../../utility/hdbTerms.ts'; +import { resolveFullTextSearchLimits } from './fullTextSearchLimits.ts'; +import { getFullTextRuntimeConfiguration, FullTextWorkerInitializationError } from './fullTextRuntimeConfiguration.ts'; import { loggerWithTag } from '../../utility/logging/logger.ts'; +const FULLTEXT_NATIVE_ABI_VERSION = 9; const FULLTEXT_LIFECYCLE_API_VERSION = 1; const FULLTEXT_MUTATION_BATCH_API_VERSION = 5; -const FULLTEXT_QUERY_API_VERSION = 3; +const FULLTEXT_QUERY_API_VERSION = 4; const logger = loggerWithTag('fulltext-derived-index'); +export interface NativeFullTextRuntimeBudget { + maxResidentIndexes: number; + maxIndexingThreads: number; + maxSearchThreads: number; + maxWriterMemoryBytes: number; + maxQueuedBytes: number; + maxExpensiveSearches: number; +} + export interface NativeFullTextIndexConfiguration { fields: Array<{ name: string; weight?: number }>; analyzer: 'english@2'; @@ -43,6 +57,12 @@ export type NativeFullTextSearchRequest = { exactTotal?: boolean; }; +export type NativeFullTextSearchPageRequest = Omit & { + continuation?: string; +}; + +export type NativeFullTextSearchPageResult = NativeFullTextSearchResult & { continuation?: string }; + export type NativeFullTextSearchResult = { total: number; totalRelation: 'exact' | 'lower-bound'; @@ -69,6 +89,10 @@ export interface NativeFullTextReader { request: NativeFullTextSearchRequest, options?: { remainingBudgetMilliseconds?: number } ): Promise; + searchPage( + request: NativeFullTextSearchPageRequest, + options?: { remainingBudgetMilliseconds?: number } + ): Promise; traceMatches( request: Omit & { text: string }, records: Array<{ id: string; fields: Record }>, @@ -98,6 +122,7 @@ export type NativeFullTextRuntimeInfo = { maxCandidateIds: number; maxCandidateBytes: number; maxSearchWindow: number; + maxSearchPageSize: number; maxAutocompleteResults: number; maxSearchResponseBytes: number; maxSearchBudgetMilliseconds: number; @@ -113,6 +138,7 @@ export type NativeFullTextQueryRuntimeInfo = Pick< limits: Pick< NativeFullTextRuntimeInfo['limits'], | 'maxSearchWindow' + | 'maxSearchPageSize' | 'maxAutocompleteResults' | 'maxRecordIdBytes' | 'maxRecordVersionBytes' @@ -126,8 +152,10 @@ export type NativeFullTextQueryRuntimeInfo = Pick< }; export interface NativeFullTextModule { + configureNativeFullTextRuntime(limits: NativeFullTextRuntimeBudget): void; NativeFullTextIndex: { - prototype: Pick; + prototype: Pick & + Pick; }; runtimeInfo(): Promise; validateNativeFullTextIndexOptions( @@ -178,7 +206,23 @@ const validatedRuntimeInfo = new WeakMap { if (!bindingPromise) { const moduleName = '@harperfast/fulltext/native'; - bindingPromise = import(moduleName).then(validateFullTextNativeBinding); + bindingPromise = import(moduleName).then(async (module) => { + const binding = await validateFullTextNativeBinding(module); + if (typeof binding.configureNativeFullTextRuntime !== 'function') + throw new TypeError('@harperfast/fulltext/native does not support process budgets'); + const runtime = resolveFullTextSearchLimits(getFullTextRuntimeConfiguration()).runtime; + binding.configureNativeFullTextRuntime(runtime); + try { + const requested = resolveFullTextSearchLimits({ + runtime: env.get(CONFIG_PARAMS.STORAGE_FULL_TEXT_RUNTIME), + }).runtime; + if (Object.keys(runtime).some((name) => runtime[name] !== requested[name])) + logger.warn?.('storage.fullText.runtime changed; restart the Harper process to apply its new budgets'); + } catch { + // A diagnostic must not reject an already-configured native binding. + } + return binding; + }); bindingPromise.catch((error) => { bindingPromise = undefined; if (!bindingWarningLogged) { @@ -186,7 +230,9 @@ export async function loadFullTextNativeBinding(): Promise const code = error && typeof error === 'object' && 'code' in error ? error.code : undefined; try { logger.warn?.( - `The @harperfast/fulltext native module is unavailable${typeof code === 'string' ? ` (${code})` : ''}` + error instanceof FullTextWorkerInitializationError + ? error.message + : `The @harperfast/fulltext native module is unavailable${typeof code === 'string' ? ` (${code})` : ''}` ); } catch {} } @@ -196,8 +242,13 @@ export async function loadFullTextNativeBinding(): Promise } export async function validateFullTextNativeBinding(module: unknown): Promise { - const prototype = (module as { NativeFullTextIndex?: { prototype?: Partial } }) - ?.NativeFullTextIndex?.prototype; + const prototype = ( + module as { + NativeFullTextIndex?: { + prototype?: Partial>; + }; + } + )?.NativeFullTextIndex?.prototype; if ( !module || typeof module !== 'object' || @@ -208,6 +259,7 @@ export async function validateFullTextNativeBinding(module: unknown): Promise runtime.limits.maxSearchWindow || diff --git a/resources/indexes/fullTextQueryIndex.ts b/resources/indexes/fullTextQueryIndex.ts index 000b5dbc5d..b20706fdde 100644 --- a/resources/indexes/fullTextQueryIndex.ts +++ b/resources/indexes/fullTextQueryIndex.ts @@ -1,4 +1,7 @@ import { createHash } from 'node:crypto'; +import * as env from '../../utility/environment/environmentManager.ts'; +import { CONFIG_PARAMS } from '../../utility/hdbTerms.ts'; +import { resolveFullTextSearchLimits } from './fullTextSearchLimits.ts'; import { fromBufferKey } from 'ordered-binary'; import { ClientError, DerivedIndexLagError, IndexRebuildingError, ServerError } from '../../utility/errors/hdbError.ts'; import { @@ -20,11 +23,12 @@ import { type NativeFullTextModule, type NativeFullTextReader, type NativeFullTextSearchExpression, + type NativeFullTextSearchPageResult, type NativeFullTextSearchMode, } from './fullTextNativeBinding.ts'; import { nativeFullTextIndexPath } from './nativeFullTextDerivedIndexLifecycle.ts'; import { decodeFullTextPublication, type FullTextPublication } from './fullTextDerivedIndex.ts'; -import { fullTextComparatorMode } from './fullTextQueryProtocol.ts'; +import { fullTextComparatorMode, replayFullTextPrefix, fullTextHitsAdvance } from './fullTextQueryProtocol.ts'; export { FULL_TEXT_QUERY_PAUSE_OPERATION, FULL_TEXT_QUERY_RESUME_OPERATION, @@ -49,10 +53,9 @@ const CANDIDATE_KEYS_PER_SOURCE_READ = 8; const MAX_CANDIDATE_KEYS = 4_096; // Four key-only reads remain cheaper than loading and decoding the authoritative record they replace. const MAX_POINT_PROBE_READS_PER_CANDIDATE = 4; -const REJECTED_GATE_SAMPLE_SIZE = 8; const CANDIDATE_GATE_FAILURE_RETRY_MILLISECONDS = 1_000; -// Fulltext 0.5.0 Query API v3 uses a 13-byte response header and 13 bytes per versioned hit before string data. -const NATIVE_SEARCH_RESPONSE_HEADER_BYTES = 13; +// Query API v4 reserves 82 traversal response bytes before hits and 13 per versioned hit before strings. +const NATIVE_SEARCH_RESPONSE_HEADER_BYTES = 82; const NATIVE_SEARCH_HIT_OVERHEAD_BYTES = 13; const MAX_RELOAD_FAILURES_BEFORE_REOPEN = 3; const HIGHLIGHT_BLOB_READ_TIMEOUT_MILLISECONDS = 5_000; @@ -266,7 +269,7 @@ export class FullTextQueryIndex { return result; }; const operation = execute().catch((error) => { - if (error === context?.signal?.reason) throw error; + if (context?.signal?.aborted && error === context.signal.reason) throw error; if (nativeErrorNeedsRebuild(error)) { this.#requestRebuild(); void this.#retireAllReaders().catch(() => undefined); @@ -358,21 +361,10 @@ export class FullTextQueryIndex { } } - async #search( - condition: FullTextCondition, - context: any, - options: { - filter?: (id: unknown, entry?: any) => boolean; - minResults?: number; - resultOffset?: number; - assertTransactionActive?: () => void; - candidateKeys?: CandidateKeyPlan; - } - ): Promise; loadedEntry: any }>> { + async #acquireSearchReader(condition: FullTextCondition, context: any) { const readiness = readDerivedIndexReadiness(this.#options.auditStore, this.#options.readinessId); assertReady(readiness, this.#definition.name); if (!this.#derivedHost) throw new IndexRebuildingError(`Full-text index '${this.#definition.name}' is not ready`); - if (options.minResults === 0) return []; const publicationGeneration = this.#publicationRevision(); const acquisitionTimeout = Math.min( MAX_WAIT_FOR_INDEX_MILLISECONDS, @@ -403,18 +395,57 @@ export class FullTextQueryIndex { } finally { if (abortListener) context.signal.removeEventListener('abort', abortListener); } + return lease; + } + + async #search( + condition: FullTextCondition, + context: any, + options: { + filter?: (id: unknown, entry?: any) => boolean; + minResults?: number; + resultOffset?: number; + assertTransactionActive?: () => void; + candidateKeys?: CandidateKeyPlan; + } + ): Promise; loadedEntry: any }>> { + if (options.minResults === 0) return []; + let policy; + try { + policy = resolveFullTextSearchLimits({ + requestBudgetMilliseconds: env.get(CONFIG_PARAMS.STORAGE_FULL_TEXT_REQUEST_BUDGET_MILLISECONDS), + maxRawHitsPerRequest: env.get(CONFIG_PARAMS.STORAGE_FULL_TEXT_MAX_RAW_HITS_PER_REQUEST), + maxResultsPerRequest: env.get(CONFIG_PARAMS.STORAGE_FULL_TEXT_MAX_RESULTS_PER_REQUEST), + }); + } catch (error) { + throw new ServerError( + `Invalid full-text configuration: ${error instanceof Error ? error.message : String(error)}`, + 500 + ); + } + const lease = await this.#acquireSearchReader(condition, context); const reader = lease.reader; try { - const deadline = performance.now() + this.#maxSearchBudgetMilliseconds!; + const deadline = + performance.now() + Math.min(policy.requestBudgetMilliseconds, this.#maxSearchBudgetMilliseconds!); + const rawHitLimit = policy.maxRawHitsPerRequest; + let rawHitsFetched = 0; const maxSearchWindow = this.#maxSearchWindow!; const query = condition.fullTextQuery ?? leafExpression(condition); const autocomplete = expressionUsesAutocomplete(query); - const searchWindow = autocomplete ? this.#maxAutocompleteResults! : maxSearchWindow; + const searchWindow = maxSearchWindow; const bounded = options.minResults !== undefined; - const target = bounded ? Math.max(1, options.minResults!) : searchWindow; - if (target > searchWindow) + const target = bounded + ? Math.max(1, options.minResults!) + : Math.min(searchWindow, rawHitLimit, policy.maxResultsPerRequest); + if (!Number.isSafeInteger(target) || target < 0 || target > rawHitLimit) throw new ClientError( - `Full-text query exceeds the ${searchWindow}-result ${autocomplete ? 'autocomplete ' : ''}search window; reduce offset or limit`, + 'Full-text offset and limit exceed storage.fullText.maxRawHitsPerRequest; reduce offset or limit, or increase the work budget', + 400 + ); + if (target > policy.maxResultsPerRequest) + throw new ClientError( + 'Full-text offset and limit exceed storage.fullText.maxResultsPerRequest; reduce offset or limit, or increase the result budget', 400 ); const accepted: Array<{ @@ -425,13 +456,11 @@ export class FullTextQueryIndex { record: Record; recordEntry: any; }> = []; - let offset = 0; + let traversal: + { continuation?: string; skip: number; prefix?: NativeFullTextSearchPageResult['hits'] } | undefined; + let maxRawPageSize = this.#maxFilteredRawPageSize!; let moreMayExist = false; - let staleVersionHits = 0; - let expiredHits = 0; - let rejectedGateSampleCount = 0; - let rejectedGateKeys: Id[] | undefined; - let rejectedGateVersions: string[] | undefined; + let previousHit: NativeFullTextSearchPageResult['hits'][number] | undefined; let rawPageSize = bounded ? Math.min( this.#maxFilteredRawPageSize!, @@ -444,39 +473,81 @@ export class FullTextQueryIndex { let candidateGate: CandidateGate | undefined; let candidateGatePlanned = false; let candidateProbeFailureEpoch = 0; - while (accepted.length < target && offset < searchWindow) { + while (accepted.length < target) { if (context?.signal?.aborted) throw context.signal.reason ?? new Error('Full-text search aborted'); - const limit = autocomplete ? searchWindow : Math.min(rawPageSize, searchWindow - offset); + const desiredLimit = traversal + ? Math.min(maxRawPageSize, rawPageSize + traversal.skip) + : autocomplete + ? this.#maxAutocompleteResults! + : Math.min(rawPageSize, maxSearchWindow); + const remainingHits = rawHitLimit - rawHitsFetched; + if (remainingHits <= 0) + throw new ClientError( + 'Full-text search exceeded storage.fullText.maxRawHitsPerRequest; narrow the query or increase the work budget', + 400 + ); + const limit = Math.min(desiredLimit, remainingHits); const acceptedBeforePage = accepted.length; - const result = await reader.search( - { - query, - offset, - limit, - ...(!bounded && offset === 0 ? { exactTotal: true } : null), - }, - { remainingBudgetMilliseconds: remainingSearchBudget(deadline) } - ); - if (!bounded && !autocomplete && result.totalRelation === 'exact' && result.total > searchWindow) - throw new ClientError(`Full-text query exceeds the ${searchWindow}-result search window; add a limit`, 400); - // Fulltext 0.5.0 Query API v3 only reports lower-bound while a full requested page remains retrievable. + let result: NativeFullTextSearchPageResult; + try { + result = traversal + ? await reader.searchPage( + { query, limit, continuation: traversal.continuation }, + { remainingBudgetMilliseconds: remainingSearchBudget(deadline) } + ) + : await reader.search( + { query, offset: 0, limit, ...(!bounded ? { exactTotal: true } : null) }, + { remainingBudgetMilliseconds: remainingSearchBudget(deadline) } + ); + } catch (error) { + if (traversal && limit > 1 && error?.code === 'E_RESULT_TOO_LARGE') { + maxRawPageSize = Math.max(1, Math.floor(limit / 2)); + rawPageSize = Math.min(rawPageSize, maxRawPageSize); + continue; + } + throw error; + } + if (context?.signal?.aborted) throw context.signal.reason ?? new Error('Full-text search aborted'); + remainingSearchBudget(deadline); + rawHitsFetched += result.hits.length; + let replayedHits = 0; + if (traversal) { + const replay = replayFullTextPrefix(traversal.prefix, traversal.skip, result); + traversal.continuation = result.continuation; + traversal.skip = replay.remaining; + replayedHits = replay.skipped; + if (traversal.skip === 0) traversal.prefix = undefined; + if (replayedHits > 0) result = { ...result, hits: replay.hits }; + } + if (!bounded && result.totalRelation === 'exact' && result.total > searchWindow) + throw unboundedSearchLimitError(target, searchWindow, rawHitLimit); + if (previousHit && result.hits.length > 0 && !fullTextHitsAdvance(previousHit, result.hits[0])) + throw new ServerError('Full-text traversal did not advance past its previous page', 500); + if (result.hits.length > 0) previousHit = result.hits[result.hits.length - 1]; + if ( + !traversal && result.totalRelation === 'lower-bound' && result.hits.length < limit && - offset + result.hits.length < result.total + result.hits.length < result.total ) { if (!this.#incompletePageFailureLogged) { this.#incompletePageFailureLogged = true; logger.error?.( - `Full-text index '${this.#definition.name}' returned ${result.hits.length} of ${limit} hits at offset ${offset} with lower-bound total ${result.total}` + `Full-text index '${this.#definition.name}' returned ${result.hits.length} of ${limit} hits at offset 0 with lower-bound total ${result.total}` ); } throw new ServerError('Full-text index returned an incomplete result page', 500); } - // An exact total can safely continue from the returned offset; only a short lower-bound page breaks the v3 contract. - moreMayExist = - result.totalRelation === 'exact' ? offset + result.hits.length < result.total : result.hits.length === limit; - if (result.hits.length === 0) break; + moreMayExist = traversal + ? result.continuation !== undefined + : result.totalRelation === 'exact' + ? result.hits.length < result.total + : result.hits.length === limit; + if (result.hits.length === 0) { + if (traversal && moreMayExist) continue; + break; + } if (!candidateGatePlanned) { candidateGatePlanned = true; if (candidatePlan) { @@ -531,14 +602,7 @@ export class FullTextQueryIndex { this.#candidateProbeFailureWarned = false; this.#candidateProbeRetryAfter = 0; } - if (!admitted) { - if (rejectedGateSampleCount < REJECTED_GATE_SAMPLE_SIZE) { - (rejectedGateKeys ??= [])[rejectedGateSampleCount] = key; - (rejectedGateVersions ??= [])[rejectedGateSampleCount] = hit.version; - rejectedGateSampleCount++; - } - continue; - } + if (!admitted) continue; candidateDecides = candidateGate.complete; } catch (error) { const usedPointProbe = candidateGate.maxReadsPerCandidate > 0; @@ -563,74 +627,40 @@ export class FullTextQueryIndex { } } } - const entry = this.#options.Table.primaryStore.getEntry(key, { transaction }); - if (!entry?.value || hit.version !== String(entry.version)) { - staleVersionHits++; - continue; - } - if (this.#options.Table.isFullTextSearchEntryCurrent?.(entry) === false) { - staleVersionHits++; - continue; - } - if (entry.expiresAt !== undefined && entry.expiresAt < Date.now()) { - expiredHits++; - continue; - } + const entry = this.#currentSearchEntry(key, hit.version, transaction); + if (!entry) continue; if (options.filter && !candidateDecides && !options.filter(key, entry)) continue; accepted.push({ key, $score: hit.score, nativeId: hit.id, record: entry.value, recordEntry: entry }); - if (accepted.length >= target) break; + if (accepted.length >= target) { + if (!bounded && hitIndex + 1 < result.hits.length) moreMayExist = true; + break; + } + } + if (!traversal && moreMayExist && accepted.length < target) { + const minimumNewHits = Math.min(target - accepted.length, Math.max(0, result.total - result.hits.length)); + if (bounded && result.hits.length + minimumNewHits > rawHitLimit - rawHitsFetched) + throw new ClientError( + 'Full-text search exceeded storage.fullText.maxRawHitsPerRequest; narrow the query or increase the work budget', + 400 + ); + traversal = { skip: result.hits.length, prefix: result.hits }; } - offset += result.hits.length; - if (autocomplete) break; if (!moreMayExist) break; if (accepted.length < target) { if (bounded) { const acceptedThisPage = accepted.length - acceptedBeforePage; rawPageSize = nextFilteredRawPageSize( - limit, + limit - replayedHits, result.hits.length, acceptedThisPage, target - accepted.length, - this.#maxFilteredRawPageSize! + maxRawPageSize ); } await new Promise((resolve) => setImmediate(resolve)); } } - if (!bounded && moreMayExist) - throw new ClientError(`Full-text query exceeds the ${searchWindow}-result search window; add a limit`, 400); - if ( - bounded && - accepted.length < target && - moreMayExist && - staleVersionHits === 0 && - rejectedGateSampleCount > 0 - ) { - options.assertTransactionActive?.(); - for (let sampleIndex = 0; sampleIndex < rejectedGateSampleCount; sampleIndex++) { - const entry = this.#options.Table.primaryStore.getEntry(rejectedGateKeys![sampleIndex], { transaction }); - if ( - !entry?.value || - rejectedGateVersions![sampleIndex] !== String(entry.version) || - this.#options.Table.isFullTextSearchEntryCurrent?.(entry) === false - ) { - staleVersionHits++; - break; - } - if (entry.expiresAt !== undefined && entry.expiresAt < Date.now()) expiredHits++; - } - } - if (bounded && accepted.length < target && moreMayExist && staleVersionHits > 0) - throw new DerivedIndexLagError( - `Full-text index '${this.#definition.name}' changed while searching; retry this query` - ); - if (bounded && accepted.length < target && moreMayExist) - throw new ClientError( - expiredHits > 0 - ? `Expired full-text matches exhausted the ${searchWindow}-result search window; retry after index cleanup or narrow the query` - : `Full-text filters exhausted the ${searchWindow}-result search window; narrow the query or reduce offset`, - 400 - ); + if (!bounded && moreMayExist) throw unboundedSearchLimitError(target, searchWindow, rawHitLimit); if (condition.includeHighlights && accepted.length > 0) await this.#addHighlights( reader, @@ -650,6 +680,14 @@ export class FullTextQueryIndex { } } + #currentSearchEntry(key: Id, version: string, transaction: any): any { + const entry = this.#options.Table.primaryStore.getEntry(key, { transaction }); + if (!entry?.value || version !== String(entry.version)) return; + if (this.#options.Table.isFullTextSearchEntryCurrent?.(entry) === false) return; + if (entry.expiresAt !== undefined && entry.expiresAt < Date.now()) return; + return entry; + } + async #candidateGate( plan: CandidateKeyPlan, target: number, @@ -890,6 +928,8 @@ export class FullTextQueryIndex { const binding = await this.#getBinding(); reader = await binding.openNativeFullTextReader(this.#nativeOptions); try { + if (typeof reader.searchPage !== 'function') + throw new TypeError('@harperfast/fulltext/native reader does not support traversal'); publication = this.#decodeReaderPublication(reader); } catch (error) { await reader.close().catch((closeError) => this.#warnReaderClose(closeError)); @@ -1068,11 +1108,11 @@ export class FullTextQueryIndex { throw new TypeError('@harperfast/fulltext/native search response limit cannot hold one maximum-size hit'); if (responseHitCapacity < info.limits.maxAutocompleteResults) logger.warn?.( - `Full-text index '${this.#definition.name}' limits autocomplete to ${responseHitCapacity} results because the native response envelope cannot hold its advertised maximum` + `Full-text index '${this.#definition.name}' limits the initial autocomplete batch to ${responseHitCapacity} results because the native response envelope cannot hold its advertised maximum` ); this.#maxFilteredRawPageSize = Math.min( MAX_FILTERED_RAW_PAGE_SIZE, - info.limits.maxSearchWindow, + info.limits.maxSearchPageSize, responseHitCapacity ); this.#maxAutocompleteResults = Math.min(info.limits.maxAutocompleteResults, responseHitCapacity); @@ -1350,6 +1390,19 @@ const REBUILD_REQUIRED_CODES = new Set([ 'E_INDEX_FORMAT_INCOMPATIBLE', ]); +function unboundedSearchLimitError(target: number, searchWindow: number, rawHitLimit: number): ClientError { + const bound = + target < searchWindow + ? target === rawHitLimit + ? 'storage.fullText.maxRawHitsPerRequest' + : 'storage.fullText.maxResultsPerRequest' + : `the ${searchWindow}-result search window`; + return new ClientError( + `Unbounded full-text query exceeds ${bound}; add a limit or increase the relevant budget`, + 400 + ); +} + function remainingSearchBudget(deadline: number): number { const remaining = Math.floor(deadline - performance.now()); if (remaining <= 0) throw new ServerError('Full-text search exceeded its execution budget', 503); diff --git a/resources/indexes/fullTextQueryProtocol.ts b/resources/indexes/fullTextQueryProtocol.ts index 2a98ce5ff2..45644f8263 100644 --- a/resources/indexes/fullTextQueryProtocol.ts +++ b/resources/indexes/fullTextQueryProtocol.ts @@ -1,3 +1,4 @@ +import { ServerError } from '../../utility/errors/hdbError.ts'; import type { NativeFullTextSearchMode } from './fullTextNativeBinding.ts'; export const FULL_TEXT_QUERY_PAUSE_OPERATION = 'pause-full-text-query-readers'; @@ -33,3 +34,43 @@ export function fullTextComparatorMode(comparator: string | undefined): NativeFu ? FULL_TEXT_MODES[comparator as PositiveFullTextComparator] : undefined; } + +type FullTextReplayHit = { id: string; score: number; version?: string }; + +function fullTextReplayPrefixMatches( + expected: readonly FullTextReplayHit[], + actual: readonly FullTextReplayHit[], + start: number, + count: number +): boolean { + for (let i = 0; i < count; i++) { + const before = expected[start + i]; + const after = actual[i]; + if (!before || !after || before.id !== after.id || before.score !== after.score || before.version !== after.version) + return false; + } + return true; +} + +export function fullTextHitsAdvance(previous: FullTextReplayHit, next: FullTextReplayHit): boolean { + return ( + previous.score > next.score || + (previous.score === next.score && Buffer.compare(Buffer.from(previous.id), Buffer.from(next.id)) < 0) + ); +} + +export function replayFullTextPrefix( + prefix: readonly FullTextReplayHit[] | undefined, + remaining: number, + page: { hits: FullTextReplayHit[]; continuation?: string } +): { skipped: number; remaining: number; hits: FullTextReplayHit[] } { + if (page.hits.length === 0 && page.continuation !== undefined) + throw new ServerError('Full-text traversal returned an empty page with more results', 500); + const skipped = Math.min(remaining, page.hits.length); + if (skipped > 0 && (!prefix || !fullTextReplayPrefixMatches(prefix, page.hits, prefix.length - remaining, skipped))) + throw new ServerError('Full-text traversal disagreed with its initial result prefix', 500); + remaining -= skipped; + if (remaining > 0 && !page.continuation) + throw new ServerError('Full-text traversal ended before its previously read hits', 500); + return { skipped, remaining, hits: skipped > 0 ? page.hits.slice(skipped) : page.hits }; +} diff --git a/resources/indexes/fullTextRuntimeConfiguration.ts b/resources/indexes/fullTextRuntimeConfiguration.ts new file mode 100644 index 0000000000..e7938cc2a7 --- /dev/null +++ b/resources/indexes/fullTextRuntimeConfiguration.ts @@ -0,0 +1,31 @@ +import { getEnvironmentData, isMainThread, setEnvironmentData, workerData } from 'node:worker_threads'; +import * as env from '../../utility/environment/environmentManager.ts'; +import { CONFIG_PARAMS } from '../../utility/hdbTerms.ts'; +import { ServerError } from '../../utility/errors/hdbError.ts'; + +const configurationKey = 'harper.fullTextRuntimeConfiguration'; +let snapshot: { runtime: unknown } | undefined = + workerData?.fullTextRuntimeConfiguration ?? getEnvironmentData(configurationKey); + +export class FullTextWorkerInitializationError extends ServerError { + readonly retryable = false; + constructor() { + super('Initialize Harper before creating application workers that use full-text search'); + this.name = 'FullTextWorkerInitializationError'; + } +} + +export function getFullTextRuntimeConfiguration(required?: true): Readonly<{ runtime: unknown }>; +export function getFullTextRuntimeConfiguration(required: false): Readonly<{ runtime: unknown }> | undefined; +export function getFullTextRuntimeConfiguration(required = true): Readonly<{ runtime: unknown }> | undefined { + if (!snapshot) { + if (!isMainThread) { + if (required) throw new FullTextWorkerInitializationError(); + return; + } + snapshot = { runtime: structuredClone(env.get(CONFIG_PARAMS.STORAGE_FULL_TEXT_RUNTIME)) }; + setEnvironmentData(configurationKey, snapshot); + } + if (snapshot.runtime && typeof snapshot.runtime === 'object') Object.freeze(snapshot.runtime); + return Object.freeze(snapshot); +} diff --git a/resources/indexes/fullTextSearchLimits.ts b/resources/indexes/fullTextSearchLimits.ts new file mode 100644 index 0000000000..b9ef33c465 --- /dev/null +++ b/resources/indexes/fullTextSearchLimits.ts @@ -0,0 +1,67 @@ +import type { NativeFullTextRuntimeBudget } from './fullTextNativeBinding.ts'; + +export type FullTextSearchLimits = Readonly<{ + requestBudgetMilliseconds: number; + maxRawHitsPerRequest: number; + maxResultsPerRequest: number; + runtime: Readonly; +}>; + +export const DEFAULT_FULL_TEXT_SEARCH_LIMITS: FullTextSearchLimits = Object.freeze({ + requestBudgetMilliseconds: 5_000, + maxRawHitsPerRequest: 10_000, + maxResultsPerRequest: 10_000, + runtime: Object.freeze({ + maxResidentIndexes: 128, + maxIndexingThreads: 128, + maxSearchThreads: 256, + maxWriterMemoryBytes: 4 * 1024 * 1024 * 1024, + maxQueuedBytes: 8 * 1024 * 1024 * 1024, + maxExpensiveSearches: 8, + }), +}); + +const policyDefaults = Object.entries(DEFAULT_FULL_TEXT_SEARCH_LIMITS).filter(([name]) => name !== 'runtime'); +const runtimeDefaults = Object.entries(DEFAULT_FULL_TEXT_SEARCH_LIMITS.runtime); + +export function resolveFullTextSearchLimits(value: unknown): FullTextSearchLimits { + if (value === undefined) return DEFAULT_FULL_TEXT_SEARCH_LIMITS; + const config = object(value, 'storage.fullText'); + const runtime = object(config.runtime, 'storage.fullText.runtime'); + const limits = {} as Record; + for (const [name, fallback] of policyDefaults) + limits[name] = positiveInteger(config[name] === undefined ? fallback : config[name], `storage.fullText.${name}`); + const budget = {} as Record; + for (const [name, fallback] of runtimeDefaults) + budget[name] = positiveInteger( + runtime[name] === undefined ? fallback : runtime[name], + `storage.fullText.runtime.${name}` + ); + for (const name of Object.keys(config)) + if (!Object.hasOwn(DEFAULT_FULL_TEXT_SEARCH_LIMITS, name)) + throw new TypeError(`Unknown storage.fullText option '${name}'`); + for (const name of Object.keys(runtime)) + if (!Object.hasOwn(DEFAULT_FULL_TEXT_SEARCH_LIMITS.runtime, name)) + throw new TypeError(`Unknown storage.fullText.runtime option '${name}'`); + for (const name of ['maxResidentIndexes', 'maxIndexingThreads', 'maxSearchThreads', 'maxExpensiveSearches']) + if (budget[name] > 0xffffffff) throw new TypeError(`storage.fullText.runtime.${name} must fit a u32`); + return Object.freeze({ ...limits, runtime: Object.freeze(budget) }) as FullTextSearchLimits; +} + +function object(value: unknown, name: string): Record { + if (value === undefined) return {}; + if ( + !value || + typeof value !== 'object' || + (Object.getPrototypeOf(value) !== Object.prototype && Object.getPrototypeOf(value) !== null) + ) + throw new TypeError(`${name} must be an object`); + return value as Record; +} + +function positiveInteger(value: unknown, name: string): number { + if (typeof value === 'string' && /^\d+$/.test(value)) value = Number(value); + if (typeof value !== 'number' || !Number.isSafeInteger(value) || value < 1) + throw new TypeError(`${name} must be a positive safe integer`); + return value; +} diff --git a/server/threads/manageThreads.ts b/server/threads/manageThreads.ts index e0a40d6f34..ea95eecee3 100644 --- a/server/threads/manageThreads.ts +++ b/server/threads/manageThreads.ts @@ -28,6 +28,7 @@ import * as hdbTerms from '../../utility/hdbTerms.ts'; import * as envMgr from '../../utility/environment/environmentManager.ts'; import harperLogger from '../../utility/logging/harper_logger.ts'; import { randomBytes } from 'node:crypto'; +import { getFullTextRuntimeConfiguration } from '../../resources/indexes/fullTextRuntimeConfiguration.ts'; import { _assignPackageExport } from '../../globals.js'; import { PACKAGE_ROOT, PRODUCT_NAME, RUNTIME_FILE_EXT } from '../../utility/packageUtils.js'; import { resolvePreloadModules } from './resolvePreload.ts'; @@ -440,6 +441,7 @@ function registerWorkerDataProvider(name?: any, provider?: any) { // setProperty() clones each value as it records it, so this provider cannot hit the log-and-skip // path below — which for this one would mean spawning the worker on the on-disk config. registerWorkerDataProvider('configOverrides', () => envMgr.getConfigOverrides()); +registerWorkerDataProvider('fullTextRuntimeConfiguration', () => getFullTextRuntimeConfiguration(false)); function collectProvidedWorkerData(options?: any) { if (workerDataProviders.size === 0) return undefined; let provided; diff --git a/server/threads/socketRouter.ts b/server/threads/socketRouter.ts index 627fd5d1b4..ae60505ac3 100644 --- a/server/threads/socketRouter.ts +++ b/server/threads/socketRouter.ts @@ -28,6 +28,7 @@ import { startHealthChecks } from '../../components/status/healthChecks.ts'; import { startTransactionLogCooling } from '../transactionLogCooling.ts'; import { applyConfiguredV8Flags } from './v8Flags.ts'; import { startLongLivedTransactionReporting } from '../../resources/longLivedTransactions.ts'; +import { getFullTextRuntimeConfiguration } from '../../resources/indexes/fullTextRuntimeConfiguration.ts'; import { setActiveWorkerPools } from './workerPools.ts'; import * as env from '../../utility/environment/environmentManager.ts'; import { isMainThread } from 'node:worker_threads'; @@ -59,6 +60,7 @@ if (isMainThread) { } export async function startHTTPThreads(threadCount = 2, dynamicThreads?: boolean) { + if (isMainThread) getFullTextRuntimeConfiguration(); applyConfiguredV8Flags(); const workerSlots = []; if (isProcessShuttingDown()) return []; diff --git a/static/defaultConfig.yaml b/static/defaultConfig.yaml index 5a070c400c..f9bfbf3983 100644 --- a/static/defaultConfig.yaml +++ b/static/defaultConfig.yaml @@ -69,6 +69,17 @@ operationsApi: securePort: 9925 rootPath: null storage: + fullText: + requestBudgetMilliseconds: 5000 + maxRawHitsPerRequest: 10000 + maxResultsPerRequest: 10000 + runtime: + maxResidentIndexes: 128 + maxIndexingThreads: 128 + maxSearchThreads: 256 + maxWriterMemoryBytes: 4294967296 + maxQueuedBytes: 8589934592 + maxExpensiveSearches: 8 writeAsync: false caching: true compression: true diff --git a/unitTests/resources/derivedIndexRuntimeRocks.test.js b/unitTests/resources/derivedIndexRuntimeRocks.test.js index 26244c674e..9db6dce433 100644 --- a/unitTests/resources/derivedIndexRuntimeRocks.test.js +++ b/unitTests/resources/derivedIndexRuntimeRocks.test.js @@ -529,6 +529,7 @@ class FakeNativeFullTextModule { applyMutationBatch() {} publish() {} close() {} + searchPage() {} }; this.states = new Map(); this.opens = []; @@ -539,11 +540,11 @@ class FakeNativeFullTextModule { return { packageVersion: 'test', tantivyVersion: 'test', - nativeAbiVersion: 5, + nativeAbiVersion: 9, queryClassIsolationMinimumSearchThreads: 2, lifecycleApiVersion: 1, mutationBatchApiVersion: 5, - queryApiVersion: 3, + queryApiVersion: 4, storageBackends: ['native'], limits: { maxCommitPayloadBytes: 64 * 1024, @@ -552,6 +553,7 @@ class FakeNativeFullTextModule { maxCandidateIds: 1_024, maxCandidateBytes: 1024 * 1024, maxSearchWindow: 10_000, + maxSearchPageSize: 10_000, maxAutocompleteResults: 100, maxSearchResponseBytes: 8 * 1024 * 1024, maxSearchBudgetMilliseconds: 30_000, diff --git a/unitTests/resources/fixtures/fullTextRuntimeConfigurationProcess.cjs b/unitTests/resources/fixtures/fullTextRuntimeConfigurationProcess.cjs new file mode 100644 index 0000000000..005005f049 --- /dev/null +++ b/unitTests/resources/fixtures/fullTextRuntimeConfigurationProcess.cjs @@ -0,0 +1,47 @@ +const env = require('#src/utility/environment/environmentManager'); +const { CONFIG_PARAMS } = require('#src/utility/hdbTerms'); +const { getFullTextRuntimeConfiguration } = require('#src/resources/indexes/fullTextRuntimeConfiguration'); +const { resolveFullTextSearchLimits } = require('#src/resources/indexes/fullTextSearchLimits'); + +async function run() { + const maxSearchThreads = Number(process.argv[2]); + env.setProperty(CONFIG_PARAMS.STORAGE_FULL_TEXT_RUNTIME, { maxSearchThreads }); + if (process.argv[3] === 'unmanaged' || process.argv[3] === 'early-worker') { + const { Worker } = require('node:worker_threads'); + const early = process.argv[3] === 'early-worker'; + if (!early) getFullTextRuntimeConfiguration(); + const result = await new Promise((resolve, reject) => { + const worker = new Worker( + ` + const { parentPort, workerData } = require('node:worker_threads'); + const env = require('#src/utility/environment/environmentManager'); + const { CONFIG_PARAMS } = require('#src/utility/hdbTerms'); + env.setProperty(CONFIG_PARAMS.STORAGE_FULL_TEXT_RUNTIME, { maxSearchThreads: workerData.maxSearchThreads + 1 }); + const optionalRuntime = require('#src/resources/indexes/fullTextRuntimeConfiguration').getFullTextRuntimeConfiguration(false); + require('#src/resources/indexes/fullTextNativeBinding').loadFullTextNativeBinding().then( + () => parentPort.postMessage('ready'), + (error) => parentPort.postMessage({ status: error.statusCode, error: error.message, name: error.name, retryable: error.retryable, optionalRuntimeAvailable: optionalRuntime !== undefined }) + ); + `, + { eval: true, workerData: { maxSearchThreads } } + ); + if (early) getFullTextRuntimeConfiguration(); + worker.once('message', resolve); + worker.once('error', reject); + }); + await require('#src/resources/indexes/fullTextNativeBinding').loadFullTextNativeBinding(); + console.log(JSON.stringify({ unmanaged: result })); + return; + } + const snapshot = getFullTextRuntimeConfiguration(); + env.setProperty(CONFIG_PARAMS.STORAGE_FULL_TEXT_RUNTIME, { maxSearchThreads: maxSearchThreads + 1 }); + if (getFullTextRuntimeConfiguration() !== snapshot) throw new Error('Runtime snapshot changed'); + const runtime = resolveFullTextSearchLimits(snapshot).runtime; + const { loadFullTextNativeBinding } = require('#src/resources/indexes/fullTextNativeBinding'); + await loadFullTextNativeBinding(); + console.log(JSON.stringify({ runtime })); +} +run().catch((error) => { + console.log(JSON.stringify({ error: error.message })); + process.exitCode = 1; +}); diff --git a/unitTests/resources/fixtures/fullTextRuntimeConfigurationWorker.js b/unitTests/resources/fixtures/fullTextRuntimeConfigurationWorker.js new file mode 100644 index 0000000000..16e764b56c --- /dev/null +++ b/unitTests/resources/fixtures/fullTextRuntimeConfigurationWorker.js @@ -0,0 +1,53 @@ +'use strict'; +const { parentPort, threadId, workerData } = require('node:worker_threads'); +workerData.noServerStart = true; +const { onMessageByType } = require('#src/server/threads/manageThreads'); +onMessageByType('full-text-runtime-test-command', async (event) => { + try { + if (event.message.operation === 'runtime-configuration') { + const { getFullTextRuntimeConfiguration } = require('#src/resources/indexes/fullTextRuntimeConfiguration'); + const { resolveFullTextSearchLimits } = require('#src/resources/indexes/fullTextSearchLimits'); + const { loadFullTextNativeBinding } = require('#src/resources/indexes/fullTextNativeBinding'); + const runtime = resolveFullTextSearchLimits(getFullTextRuntimeConfiguration()).runtime; + const binding = await loadFullTextNativeBinding(); + binding.configureNativeFullTextRuntime(runtime); + let changedCode; + try { + binding.configureNativeFullTextRuntime({ ...runtime, maxSearchThreads: runtime.maxSearchThreads + 1 }); + } catch (error) { + changedCode = error.code; + } + const { Worker } = require('node:worker_threads'); + const { once } = require('node:events'); + const child = new Worker( + ` + const { parentPort, workerData } = require('node:worker_threads'); + const env = require('#src/utility/environment/environmentManager'); + const { CONFIG_PARAMS } = require('#src/utility/hdbTerms'); + env.setProperty(CONFIG_PARAMS.STORAGE_FULL_TEXT_RUNTIME, { maxSearchThreads: workerData.maxSearchThreads + 1 }); + require('#src/resources/indexes/fullTextNativeBinding').loadFullTextNativeBinding().then( + () => parentPort.postMessage({ ready: true }), + (error) => parentPort.postMessage({ error: error.message }) + ); + `, + { eval: true, workerData: { maxSearchThreads: runtime.maxSearchThreads } } + ); + const [rawChild] = await once(child, 'message', { signal: AbortSignal.timeout(10000) }).finally(() => + child.terminate() + ); + if (!rawChild.ready) throw new Error(rawChild.error); + parentPort.postMessage({ + type: 'full-text-runtime-test-response', + message: { originator: threadId, value: { runtime, changedCode, rawChildReady: rawChild.ready } }, + }); + return; + } + } catch (error) { + parentPort.postMessage({ + type: 'full-text-runtime-test-response', + message: { originator: threadId, error: String(error) }, + }); + } +}); +parentPort.ref(); +parentPort.postMessage({ type: 'full-text-runtime-test-response', message: { originator: threadId, ready: true } }); diff --git a/unitTests/resources/fullTextNativePlatform.js b/unitTests/resources/fullTextNativePlatform.js new file mode 100644 index 0000000000..83570e1ea0 --- /dev/null +++ b/unitTests/resources/fullTextNativePlatform.js @@ -0,0 +1,13 @@ +async function supportsPublishedFullTextBinding() { + const glibc = process.platform === 'linux' && process.report?.getReport().header.glibcVersionRuntime; + if (!( + (process.platform === 'darwin' && process.arch === 'arm64') || + (process.platform === 'linux' && glibc && ['arm64', 'x64'].includes(process.arch)) || + (process.platform === 'win32' && process.arch === 'x64') + )) + return false; + await import('@harperfast/fulltext/native'); + return true; +} + +module.exports = { supportsPublishedFullTextBinding }; diff --git a/unitTests/resources/fullTextNativeTestBinding.js b/unitTests/resources/fullTextNativeTestBinding.js index 5f1e58c088..cec72221ab 100644 --- a/unitTests/resources/fullTextNativeTestBinding.js +++ b/unitTests/resources/fullTextNativeTestBinding.js @@ -22,6 +22,7 @@ class FullTextNativeTestBinding { applyMutationBatch() {} publish() {} close() {} + searchPage() {} }; } @@ -29,11 +30,11 @@ class FullTextNativeTestBinding { return { packageVersion: 'test', tantivyVersion: 'test', - nativeAbiVersion: 5, + nativeAbiVersion: 9, queryClassIsolationMinimumSearchThreads: 2, lifecycleApiVersion: 1, mutationBatchApiVersion: 5, - queryApiVersion: 3, + queryApiVersion: 4, storageBackends: ['native'], limits: { maxCommitPayloadBytes: 64 * 1024, @@ -42,6 +43,7 @@ class FullTextNativeTestBinding { maxCandidateIds: 1_024, maxCandidateBytes: 1024 * 1024, maxSearchWindow: 10_000, + maxSearchPageSize: 10_000, maxAutocompleteResults: 100, maxSearchResponseBytes: 8 * 1024 * 1024, maxSearchBudgetMilliseconds: 30_000, @@ -128,6 +130,12 @@ class FullTextNativeTestBinding { hits: hits.slice(offset, offset + limit), }; }, + async searchPage({ continuation, ...request }, options) { + const offset = Number(continuation ?? 0); + const result = await this.search({ ...request, offset }, options); + const next = offset + result.hits.length; + return { ...result, continuation: next < result.total ? String(next) : undefined }; + }, async traceMatches(request, records) { validateTraceRequest(request); const terms = String(request.text).toLowerCase().split(/\s+/).filter(Boolean); diff --git a/unitTests/resources/fullTextQueryIndex.test.js b/unitTests/resources/fullTextQueryIndex.test.js index 0e5daa7e57..6d8ea0cc93 100644 --- a/unitTests/resources/fullTextQueryIndex.test.js +++ b/unitTests/resources/fullTextQueryIndex.test.js @@ -1,3 +1,4 @@ +const { getFullTextTestConfiguration, setFullTextTestConfiguration } = require('./fullTextTestConfiguration'); require('../testUtils'); const assert = require('node:assert'); const { toBufferKey } = require('ordered-binary'); @@ -13,6 +14,22 @@ const { const { encodeFullTextCursorPayload } = require('#src/resources/indexes/fullTextDerivedIndex'); const { nativeFullTextIndexPath } = require('#src/resources/indexes/nativeFullTextDerivedIndexLifecycle'); +function withTraversal(reader) { + if (reader?.then) return reader.then(withTraversal); + if (typeof reader.searchPage === 'function') return reader; + reader.searchPage = async function ({ continuation, ...request }, options) { + const offset = Number(continuation ?? 0); + const result = await this.search({ ...request, offset }, options); + const next = offset + result.hits.length; + return { + ...result, + totalRelation: 'exact', + continuation: result.hits.length > 0 && next < result.total ? String(next) : undefined, + }; + }; + return reader; +} + function sharedStore() { const buffers = new Map(); const callbacks = new Map(); @@ -113,6 +130,7 @@ function queryLimits(overrides = {}) { maxCandidateIds: 1_024, maxCandidateBytes: 1024 * 1024, maxSearchWindow, + maxSearchPageSize: maxSearchWindow, maxAutocompleteResults: Math.min(100, maxSearchWindow), maxSearchResponseBytes: 8 * 1024 * 1024, maxSearchBudgetMilliseconds: 30_000, @@ -140,6 +158,8 @@ function simpleQueryIndex({ hits, onReload, onClose, + onOpen, + onTrace, onGetEntry, onSearch, entryForKey, @@ -149,6 +169,7 @@ function simpleQueryIndex({ sourceGeneration = 'generation', maxSearchWindow = 10, searchResult, + searchPageResult, }) { let committedPayload = payload; let reloads = 0; @@ -182,7 +203,9 @@ function simpleQueryIndex({ }; }, async openNativeFullTextReader() { - return { + const readerId = onOpen?.(); + return withTraversal({ + ...(onTrace ? { traceMatches: (request, records) => onTrace(request, records, readerId) } : null), get committedPayload() { return committedPayload; }, @@ -193,6 +216,13 @@ function simpleQueryIndex({ const current = hits(); return { total: current.length, totalRelation: 'exact', hits: current.slice(offset, offset + limit) }; }, + async searchPage({ continuation, ...request }) { + if (searchPageResult) return searchPageResult({ continuation, ...request }); + const offset = Number(continuation ?? 0); + const result = await this.search({ ...request, offset }); + const next = offset + result.hits.length; + return { ...result, continuation: next < result.total ? String(next) : undefined }; + }, async reload() { reloads++; onReload?.(); @@ -200,7 +230,7 @@ function simpleQueryIndex({ async close() { onClose?.(); }, - }; + }); }, }, }); @@ -226,7 +256,301 @@ function readyQueryIndex(auditStore, readinessId, storeName) { return fixture.index; } +describe('full-text replay page protocol', () => { + const { replayFullTextPrefix } = require('#src/resources/indexes/fullTextQueryProtocol'); + const prefix = [ + { id: 'a', score: 3, version: '1' }, + { id: 'b', score: 2, version: '1' }, + { id: 'c', score: 1, version: '1' }, + ]; + it('accounts for a prefix split over several lower-bound pages before returning new hits', () => { + const first = { hits: prefix.slice(0, 1), totalRelation: 'lower-bound', continuation: 'next' }; + const one = replayFullTextPrefix(prefix, 3, first); + assert.deepStrictEqual(one, { skipped: 1, remaining: 2, hits: [] }); + assert.strictEqual(first.totalRelation, 'lower-bound'); + const next = { id: 'd', score: 0, version: '1' }; + assert.deepStrictEqual(replayFullTextPrefix(prefix, one.remaining, { hits: [...prefix.slice(1), next] }), { + skipped: 2, + remaining: 0, + hits: [next], + }); + }); + it('rejects an empty page claiming more hits, a changed prefix and premature exhaustion', () => { + for (const page of [ + { hits: [], continuation: 'next' }, + { hits: [{ ...prefix[0], version: '2' }], continuation: 'next' }, + { hits: prefix.slice(0, 1) }, + ]) + assert.throws( + () => replayFullTextPrefix(prefix, 3, page), + (error) => error.statusCode === 500 + ); + assert.deepStrictEqual(replayFullTextPrefix(undefined, 0, { hits: [] }), { skipped: 0, remaining: 0, hits: [] }); + }); +}); + +describe('full-text traversal ordering guard', () => { + it('requires a new page to advance by score and then UTF-8 ID order', () => { + const { fullTextHitsAdvance } = require('#src/resources/indexes/fullTextQueryProtocol'); + assert.strictEqual(fullTextHitsAdvance({ id: 'z', score: 2 }, { id: 'a', score: 1 }), true); + assert.strictEqual(fullTextHitsAdvance({ id: '\uE000', score: 1 }, { id: '😀', score: 1 }), true); + assert.strictEqual(fullTextHitsAdvance({ id: 'a', score: 1 }, { id: 'a', score: 1 }), false); + assert.strictEqual(fullTextHitsAdvance({ id: 'b', score: 1 }, { id: 'a', score: 1 }), false); + assert.strictEqual(fullTextHitsAdvance({ id: 'a', score: 1 }, { id: 'b', score: 2 }), false); + }); +}); + describe('FullTextQueryIndex', () => { + it('keeps a filled ordinary request out of native traversal admission', async () => { + const auditStore = sharedStore(); + const readinessId = 'ordinary-query-admission'; + publishDerivedIndexReadiness(auditStore, readinessId, 'ready'); + const { index } = simpleQueryIndex({ + auditStore, + readinessId, + payload: publicationPayload(), + hits: () => [{ id: nativeId(1, 'one'), score: 1, version: '1' }], + searchPageResult: () => { + throw new Error('ordinary query must not enter traversal'); + }, + }); + attachCurrentCoverage(index, auditStore, readinessId); + assert.strictEqual((await index.search({ comparator: 'matches', value: 'shoe' }, {}, { minResults: 1 })).length, 1); + await index.close(); + }); + + for (const filtered of [false, true]) + it(`rejects impossible replay work before traversal (${filtered ? 'filtered' : 'ordinary'})`, async () => { + const previous = getFullTextTestConfiguration(); + const auditStore = sharedStore(); + const readinessId = `impossible-replay-${filtered}`; + publishDerivedIndexReadiness(auditStore, readinessId, 'ready'); + const hits = Array.from({ length: 1000 }, (_, i) => ({ + id: nativeId(1, `record-${i}`), + version: '1', + score: 1000 - i, + })); + let searches = 0; + const { index } = simpleQueryIndex({ + auditStore, + readinessId, + payload: publicationPayload(), + maxSearchWindow: 10000, + hits: () => hits, + onSearch: () => searches++, + }); + attachCurrentCoverage(index, auditStore, readinessId); + try { + setFullTextTestConfiguration({ ...previous, maxRawHitsPerRequest: 500 }); + await assert.rejects( + index.search( + { comparator: 'matches', value: 'shoe' }, + {}, + { + minResults: 300, + ...(filtered ? { filter: () => false } : null), + } + ), + (error) => error.statusCode === 400 && /maxRawHitsPerRequest/.test(error.message) + ); + assert.strictEqual(searches, 1); + } finally { + setFullTextTestConfiguration(previous); + await index.close(); + } + }); + + it('allows exhausted matches below the target when replay still fits the work budget', async () => { + const previous = getFullTextTestConfiguration(); + const auditStore = sharedStore(); + const readinessId = 'exhausted-replay-budget'; + publishDerivedIndexReadiness(auditStore, readinessId, 'ready'); + const hits = Array.from({ length: 260 }, (_, i) => ({ + id: nativeId(1, `record-${i}`), + version: '1', + score: 260 - i, + })); + const { index } = simpleQueryIndex({ + auditStore, + readinessId, + payload: publicationPayload(), + maxSearchWindow: 10000, + hits: () => hits, + }); + attachCurrentCoverage(index, auditStore, readinessId); + try { + setFullTextTestConfiguration({ ...previous, maxRawHitsPerRequest: 600 }); + const result = await index.search({ comparator: 'matches', value: 'shoe' }, {}, { minResults: 500 }); + assert.strictEqual(result.length, 260); + } finally { + setFullTextTestConfiguration(previous); + await index.close(); + } + }); + + for (const [count, budget, expectedCalls] of [ + [1000, 500, 1], + [256, 512, 2], + [1000, 550, 2], + ]) + it(`bounds replay with an uninformative lower-bound total (${count} hits, budget ${budget})`, async () => { + const previous = getFullTextTestConfiguration(); + const auditStore = sharedStore(); + const readinessId = `lower-bound-replay-${count}-${budget}`; + publishDerivedIndexReadiness(auditStore, readinessId, 'ready'); + const hits = Array.from({ length: count }, (_, i) => ({ + id: nativeId(1, `record-${i}`), + version: '1', + score: count - i, + })); + let searches = 0; + const { index } = simpleQueryIndex({ + auditStore, + readinessId, + payload: publicationPayload(), + maxSearchWindow: 10000, + hits: () => hits, + searchResult: ({ limit }) => { + searches++; + return { total: limit, totalRelation: 'lower-bound', hits: hits.slice(0, limit) }; + }, + searchPageResult: ({ continuation, limit }) => { + searches++; + const offset = Number(continuation ?? 0); + const page = hits.slice(offset, offset + limit); + const next = offset + page.length; + return { + total: count, + totalRelation: 'exact', + hits: page, + continuation: next < count ? String(next) : undefined, + }; + }, + }); + attachCurrentCoverage(index, auditStore, readinessId); + try { + setFullTextTestConfiguration({ ...previous, maxRawHitsPerRequest: budget }); + const search = index.search({ comparator: 'matches', value: 'shoe' }, {}, { minResults: 300 }); + if (count < 300) assert.strictEqual((await search).length, count); + else + await assert.rejects( + search, + (error) => error.statusCode === 400 && /maxRawHitsPerRequest/.test(error.message) + ); + assert.strictEqual(searches, expectedCalls); + } finally { + setFullTextTestConfiguration(previous); + await index.close(); + } + }); + + it('rejects an empty native page claiming a continuation', async () => { + const auditStore = sharedStore(); + const readinessId = 'empty-traversal-page'; + publishDerivedIndexReadiness(auditStore, readinessId, 'ready'); + const { index } = simpleQueryIndex({ + auditStore, + readinessId, + payload: publicationPayload(), + hits: () => [], + searchResult: () => ({ + total: 2, + totalRelation: 'exact', + hits: [{ id: nativeId(1, 'one'), score: 1, version: '1' }], + }), + searchPageResult: () => ({ total: 1, totalRelation: 'exact', hits: [], continuation: 'next' }), + }); + attachCurrentCoverage(index, auditStore, readinessId); + await assert.rejects( + index.search( + { attribute: readinessId, comparator: 'matches', value: 'pack' }, + {}, + { minResults: 1, filter: () => false } + ), + /empty page with more results/ + ); + await index.close(); + }); + + it('keeps retained result limits independent from raw work and reports invalid configuration', async () => { + const previous = getFullTextTestConfiguration(); + const auditStore = sharedStore(); + const readinessId = 'independent-results-budget'; + publishDerivedIndexReadiness(auditStore, readinessId, 'ready'); + let searches = 0; + const { index } = simpleQueryIndex({ + auditStore, + readinessId, + payload: publicationPayload(), + hits: () => [{ id: nativeId(1, 'one'), version: '1', score: 1 }], + onSearch: () => searches++, + }); + attachCurrentCoverage(index, auditStore, readinessId); + try { + setFullTextTestConfiguration({ + ...previous, + maxRawHitsPerRequest: 500000, + maxResultsPerRequest: 10, + }); + await assert.rejects( + index.search({ comparator: 'matches', value: 'shoe' }, {}, { minResults: 11 }), + (error) => error.statusCode === 400 && /maxResultsPerRequest/.test(error.message) + ); + assert.strictEqual(searches, 0); + setFullTextTestConfiguration({ ...previous, maxRawHitsPerRequest: 0 }); + await assert.rejects( + index.search({ comparator: 'matches', value: 'shoe' }, {}, { minResults: 1 }), + (error) => + error.statusCode === 500 && /Invalid full-text configuration:.*maxRawHitsPerRequest/.test(error.message) + ); + assert.strictEqual(searches, 0); + } finally { + setFullTextTestConfiguration(previous); + await index.close(); + } + }); + + it('rejects unbounded results at the actual work or retained-result ceiling', async () => { + const previous = getFullTextTestConfiguration(); + const auditStore = sharedStore(); + const readinessId = 'unbounded-result-budget'; + publishDerivedIndexReadiness(auditStore, readinessId, 'ready'); + let nativeHitCount = 200; + const { index } = simpleQueryIndex({ + auditStore, + readinessId, + payload: publicationPayload(), + maxSearchWindow: 1000, + hits: () => + Array.from({ length: nativeHitCount }, (_, i) => ({ + id: nativeId(1, String(i)), + score: 200 - i, + version: '1', + })), + }); + attachCurrentCoverage(index, auditStore, readinessId); + try { + for (const count of [200, 2000]) { + nativeHitCount = count; + for (const name of ['maxRawHitsPerRequest', 'maxResultsPerRequest']) { + setFullTextTestConfiguration({ + ...previous, + maxRawHitsPerRequest: 500, + maxResultsPerRequest: 500, + [name]: 100, + }); + await assert.rejects( + index.search({ comparator: 'matches', value: 'shoe' }, {}), + (error) => error.statusCode === 400 && error.message.includes(`storage.fullText.${name}`) + ); + } + } + } finally { + setFullTextTestConfiguration(previous); + await index.close(); + } + }); + it('does not treat prototype property names as full-text comparators', () => { for (const comparator of ['constructor', 'toString', '__proto__']) assert.strictEqual(fullTextComparatorMode(comparator), undefined); @@ -376,7 +700,7 @@ describe('FullTextQueryIndex', () => { }, async openNativeFullTextReader(options) { readerOptions = options; - return reader; + return withTraversal(reader); }, }; const Table = { @@ -488,7 +812,7 @@ describe('FullTextQueryIndex', () => { }; }, async openNativeFullTextReader() { - return { + return withTraversal({ get committedPayload() { return payload; }, @@ -507,7 +831,7 @@ describe('FullTextQueryIndex', () => { } }, async close() {}, - }; + }); }, }, }); @@ -710,14 +1034,14 @@ describe('FullTextQueryIndex', () => { isEntryCurrent: (entry) => entry.metadataFlags === 0, }); attachCurrentCoverage(index, auditStore, readinessId); - await assert.rejects( - index.search({ attribute: readinessId, comparator: 'matches', value: 'shoe' }, {}, { minResults: 1 }), - (error) => error.name === 'DerivedIndexLagError' + assert.deepStrictEqual( + await index.search({ attribute: readinessId, comparator: 'matches', value: 'shoe' }, {}, { minResults: 1 }), + [] ); await index.close(); }); - it('reports an expired bounded native window as expired matches', async () => { + it('exhausts expired native matches beyond the ordinary window', async () => { const auditStore = sharedStore(); const readinessId = 'expired-native-window'; publishDerivedIndexReadiness(auditStore, readinessId, 'ready'); @@ -729,11 +1053,9 @@ describe('FullTextQueryIndex', () => { entryForKey: (key) => ({ version: 1, value: { title: key }, expiresAt: Date.now() - 1 }), }); attachCurrentCoverage(index, auditStore, readinessId); - await assert.rejects( - index.search({ attribute: readinessId, comparator: 'matches', value: 'shoe' }, {}, { minResults: 1 }), - (error) => - error.statusCode === 400 && - /Expired full-text matches exhausted the 10-result search window/.test(error.message) + assert.deepStrictEqual( + await index.search({ attribute: readinessId, comparator: 'matches', value: 'shoe' }, {}, { minResults: 1 }), + [] ); await index.close(); }); @@ -854,14 +1176,14 @@ describe('FullTextQueryIndex', () => { }; }, async openNativeFullTextReader() { - return { + return withTraversal({ committedPayload: payload, async search() {}, async reload() {}, async close() { closes++; }, - }; + }); }, }, }); @@ -936,7 +1258,7 @@ describe('FullTextQueryIndex', () => { }; }, async openNativeFullTextReader() { - return { committedPayload: undefined, async search() {}, async reload() {}, async close() {} }; + return withTraversal({ committedPayload: undefined, async search() {}, async reload() {}, async close() {} }); }, }, }); @@ -980,7 +1302,7 @@ describe('FullTextQueryIndex', () => { }; }, async openNativeFullTextReader() { - return { committedPayload: undefined, async search() {}, async reload() {}, async close() {} }; + return withTraversal({ committedPayload: undefined, async search() {}, async reload() {}, async close() {} }); }, }, }); @@ -1055,14 +1377,14 @@ describe('FullTextQueryIndex', () => { }, async openNativeFullTextReader() { opens++; - return { + return withTraversal({ async search() { return { total: 0, totalRelation: 'exact', hits: [] }; }, committedPayload: publicationPayload(), async reload() {}, async close() {}, - }; + }); }, }, }); @@ -1149,7 +1471,7 @@ describe('FullTextQueryIndex', () => { }, async openNativeFullTextReader() { opens++; - return { + return withTraversal({ async search(searchRequest) { request = searchRequest; return { total: 0, totalRelation: 'exact', hits: [] }; @@ -1157,7 +1479,7 @@ describe('FullTextQueryIndex', () => { committedPayload: publicationPayload(), async reload() {}, async close() {}, - }; + }); }, }, }); @@ -1171,19 +1493,19 @@ describe('FullTextQueryIndex', () => { } }; await assert.rejects( - search('response-cannot-hold-hit', { maxSearchResponseBytes: 27 }, 'matches', 0), + search('response-cannot-hold-hit', { maxSearchResponseBytes: 96 }, 'matches', 0), (error) => error.statusCode === 500 && error.message === "Full-text search on 'catalogSearch' failed" ); - assert.deepStrictEqual((await search('response-clamps-autocomplete', { maxSearchResponseBytes: 42 })).results, []); + assert.deepStrictEqual((await search('response-clamps-autocomplete', { maxSearchResponseBytes: 111 })).results, []); const autocomplete = await search( 'response-clamps-autocomplete-query', - { maxSearchResponseBytes: 42 }, + { maxSearchResponseBytes: 111 }, 'matches_prefix' ); assert.deepStrictEqual(autocomplete.results, []); assert.strictEqual(autocomplete.request.limit, 1); assert.deepStrictEqual( - (await search('response-autocomplete-boundary', { maxSearchResponseBytes: 43 })).results, + (await search('response-autocomplete-boundary', { maxSearchResponseBytes: 112 })).results, [] ); }); @@ -1213,14 +1535,14 @@ describe('FullTextQueryIndex', () => { }; }, async openNativeFullTextReader() { - return { + return withTraversal({ async search() { throw errors.shift(); }, committedPayload: publicationPayload(), async reload() {}, async close() {}, - }; + }); }, }, }); @@ -1254,7 +1576,7 @@ describe('FullTextQueryIndex', () => { }, async openNativeFullTextReader(options) { opens.push(structuredClone(options)); - return { + return withTraversal({ async search() { return { total: 1, @@ -1272,7 +1594,7 @@ describe('FullTextQueryIndex', () => { async close() { closes++; }, - }; + }); }, }; const index = new FullTextQueryIndex({ @@ -1338,7 +1660,7 @@ describe('FullTextQueryIndex', () => { }, async openNativeFullTextReader() { opens++; - return { + return withTraversal({ async search() { return { total: 1, @@ -1353,7 +1675,7 @@ describe('FullTextQueryIndex', () => { async close() { closes++; }, - }; + }); }, }, }); @@ -1373,7 +1695,7 @@ describe('FullTextQueryIndex', () => { await index.close(); }); - it('requires callers to page searches larger than the native result window', async () => { + it('bounds unbounded searches but permits bounded requests beyond the native result window', async () => { const auditStore = sharedStore(); publishDerivedIndexReadiness(auditStore, 'query-window', 'ready'); let searches = 0; @@ -1402,7 +1724,7 @@ describe('FullTextQueryIndex', () => { }; }, async openNativeFullTextReader() { - return { + return withTraversal({ async search(request) { searches++; lastRequest = request; @@ -1412,13 +1734,14 @@ describe('FullTextQueryIndex', () => { hits: [ { id: nativeId(1, 'one'), version: '1', score: 2 }, { id: nativeId(1, 'two'), version: '1', score: 1 }, - ], + { id: nativeId(1, 'three'), version: '1', score: 0 }, + ].slice(request.offset ?? 0, (request.offset ?? 0) + request.limit), }; }, committedPayload: publicationPayload(), async reload() {}, async close() {}, - }; + }); }, }, }); @@ -1426,8 +1749,7 @@ describe('FullTextQueryIndex', () => { const query = { attribute: 'query-window', comparator: 'matches', value: 'shoe' }; assert.deepStrictEqual(await index.search(query, {}, { minResults: 0 }), []); assert.strictEqual(searches, 0); - await assert.rejects(index.search(query, {}, { minResults: 3 }), /reduce offset or limit/); - assert.strictEqual(searches, 0); + assert.strictEqual((await index.search(query, {}, { minResults: 3 })).length, 3); await assert.rejects(index.search(query, {}), /exceeds the 2-result search window/); assert.strictEqual(lastRequest.exactTotal, true); await index.close(); @@ -1465,7 +1787,7 @@ describe('FullTextQueryIndex', () => { }; }, async openNativeFullTextReader() { - return { + return withTraversal({ async search(request) { requests.push(request); return { @@ -1477,7 +1799,7 @@ describe('FullTextQueryIndex', () => { committedPayload: publicationPayload(), async reload() {}, async close() {}, - }; + }); }, }, }); @@ -1488,7 +1810,7 @@ describe('FullTextQueryIndex', () => { requests.map(({ offset, limit, exactTotal }) => ({ offset, limit, exactTotal })), [ { offset: 0, limit: 256, exactTotal: true }, - { offset: 256, limit: 256, exactTotal: undefined }, + { offset: 0, limit: 512, exactTotal: undefined }, ] ); await index.close(); @@ -1525,7 +1847,7 @@ describe('FullTextQueryIndex', () => { requests.map(({ offset, limit }) => ({ offset, limit })), [ { offset: 0, limit: 40 }, - { offset: 40, limit: 160 }, + { offset: 0, limit: 200 }, { offset: 200, limit: 640 }, { offset: 840, limit: 1_022 }, { offset: 1_862, limit: 1_022 }, @@ -1570,7 +1892,7 @@ describe('FullTextQueryIndex', () => { requests.map(({ offset, limit }) => ({ offset, limit })), [ { offset: 0, limit: 40 }, - { offset: 40, limit: 720 }, + { offset: 0, limit: 760 }, ] ); assert.deepStrictEqual( @@ -2509,7 +2831,7 @@ describe('FullTextQueryIndex', () => { await index.close(); }); - it('uses the native autocomplete window for prefix expressions', async () => { + it('keeps the initial autocomplete window and traverses larger prefix expressions', async () => { const auditStore = sharedStore(); const readinessId = 'autocomplete-window'; publishDerivedIndexReadiness(auditStore, readinessId, 'ready'); @@ -2541,19 +2863,19 @@ describe('FullTextQueryIndex', () => { }; }, async openNativeFullTextReader() { - return { + return withTraversal({ async search(request) { requests.push(request); return { total: hits.length, totalRelation: 'exact', - hits: hits.slice(0, request.limit), + hits: hits.slice(request.offset, request.offset + request.limit), }; }, committedPayload: publicationPayload(), async reload() {}, async close() {}, - }; + }); }, }, }); @@ -2570,75 +2892,88 @@ describe('FullTextQueryIndex', () => { ], }, }; - await assert.rejects(index.search(query, {}), /100-result search window; add a limit/); - assert.deepStrictEqual( - requests.map(({ offset, limit, exactTotal }) => ({ offset, limit, exactTotal })), - [{ offset: 0, limit: 100, exactTotal: true }] - ); + assert.strictEqual((await index.search(query, {})).length, hits.length); + assert.deepStrictEqual((({ offset, limit, exactTotal }) => ({ offset, limit, exactTotal }))(requests[0]), { + offset: 0, + limit: 100, + exactTotal: true, + }); + requests.length = 0; assert.strictEqual((await index.search(query, {}, { minResults: 100 })).length, 100); - await assert.rejects(index.search(query, {}, { minResults: 101 }), /100-result autocomplete search window/); - assert.strictEqual(requests.length, 2); + assert.strictEqual(requests.length, 1); + assert.strictEqual((await index.search(query, {}, { minResults: 101 })).length, 101); await index.close(); }); - it('applies one execution budget across every native result page', async () => { - const auditStore = sharedStore(); - const readinessId = 'query-budget'; - publishDerivedIndexReadiness(auditStore, readinessId, 'ready'); - let searches = 0; - const index = new FullTextQueryIndex({ - Table: { - tableId: 1, - primaryStore: { getEntry: (key) => ({ version: 1, value: { title: key } }) }, - _readTxnForContext: () => undefined, - }, - definition: definition(), - auditStore, - readinessId, - indexId: readinessId, - storePath: '/unused', - storeName: 'unused', - sourceGeneration: 'generation', - limits: {}, - binding: { - async runtimeInfo() { - return { - queryClassIsolationMinimumSearchThreads: 1, - limits: queryLimits({ maxSearchWindow: 10_000, maxSearchBudgetMilliseconds: 20, maxTraceRecords: 10 }), - }; - }, - async openNativeFullTextReader() { - return { - async search(request) { - searches++; - await new Promise((resolve) => setTimeout(resolve, 25)); + for (const configured of [false, true]) + it(`applies the ${configured ? 'Harper' : 'native'} execution budget across every result page`, async () => { + const previous = getFullTextTestConfiguration(); + if (configured) setFullTextTestConfiguration({ ...previous, requestBudgetMilliseconds: 20 }); + try { + const auditStore = sharedStore(); + const readinessId = 'query-budget'; + publishDerivedIndexReadiness(auditStore, readinessId, 'ready'); + let searches = 0; + const index = new FullTextQueryIndex({ + Table: { + tableId: 1, + primaryStore: { getEntry: (key) => ({ version: 1, value: { title: key } }) }, + _readTxnForContext: () => undefined, + }, + definition: definition(), + auditStore, + readinessId, + indexId: readinessId, + storePath: '/unused', + storeName: 'unused', + sourceGeneration: 'generation', + limits: {}, + binding: { + async runtimeInfo() { return { - total: 300, - totalRelation: 'exact', - hits: Array.from({ length: request.limit }, (_value, offset) => ({ - id: nativeId(1, `record-${request.offset + offset}`), - version: '1', - score: 300 - request.offset - offset, - })), + queryClassIsolationMinimumSearchThreads: 1, + limits: queryLimits({ + maxSearchWindow: 10_000, + maxSearchBudgetMilliseconds: configured ? 30000 : 20, + maxTraceRecords: 10, + }), }; }, - committedPayload: publicationPayload(), - async reload() {}, - async close() {}, - }; - }, - }, + async openNativeFullTextReader() { + return withTraversal({ + async search(request) { + searches++; + await new Promise((resolve) => setTimeout(resolve, 25)); + return { + total: 300, + totalRelation: 'exact', + hits: Array.from({ length: request.limit }, (_value, offset) => ({ + id: nativeId(1, `record-${request.offset + offset}`), + version: '1', + score: 300 - request.offset - offset, + })), + }; + }, + committedPayload: publicationPayload(), + async reload() {}, + async close() {}, + }); + }, + }, + }); + attachCurrentCoverage(index, auditStore, readinessId); + await assert.rejects( + index.search({ attribute: readinessId, comparator: 'matches', value: 'shoe' }, {}), + (error) => error.statusCode === 503 && /execution budget/.test(error.message) + ); + assert.strictEqual(searches, 1); + await index.close(); + } finally { + setFullTextTestConfiguration(previous); + } }); - attachCurrentCoverage(index, auditStore, readinessId); - await assert.rejects( - index.search({ attribute: readinessId, comparator: 'matches', value: 'shoe' }, {}), - (error) => error.statusCode === 503 && /execution budget/.test(error.message) - ); - assert.strictEqual(searches, 1); - await index.close(); - }); - it('fails instead of returning an incomplete bounded page after filtering exhausts the native window', async () => { + it('returns an exhausted filtered page after traversing all raw results', async () => { const auditStore = sharedStore(); publishDerivedIndexReadiness(auditStore, 'filtered-window', 'ready'); const index = new FullTextQueryIndex({ @@ -2663,7 +2998,7 @@ describe('FullTextQueryIndex', () => { }; }, async openNativeFullTextReader() { - return { + return withTraversal({ async search() { return { total: 2, @@ -2677,23 +3012,23 @@ describe('FullTextQueryIndex', () => { committedPayload: publicationPayload(), async reload() {}, async close() {}, - }; + }); }, }, }); attachCurrentCoverage(index, auditStore, 'filtered-window'); - await assert.rejects( - index.search( + assert.deepStrictEqual( + await index.search( { attribute: 'filtered-window', comparator: 'matches', value: 'shoe' }, {}, { minResults: 1, filter: () => false } ), - /filters exhausted the 2-result search window/ + [] ); await index.close(); }); - it('reports stale native versions as retryable index lag', async () => { + it('returns no stale native versions after exhausting the snapshot', async () => { const auditStore = sharedStore(); publishDerivedIndexReadiness(auditStore, 'stale-window', 'ready'); const index = new FullTextQueryIndex({ @@ -2718,27 +3053,28 @@ describe('FullTextQueryIndex', () => { }; }, async openNativeFullTextReader() { - return { - async search() { + return withTraversal({ + async search(request) { return { total: 3, totalRelation: 'lower-bound', hits: [ { id: nativeId(1, 'one'), version: '1', score: 2 }, { id: nativeId(1, 'two'), version: '1', score: 1 }, - ], + { id: nativeId(1, 'three'), version: '1', score: 0 }, + ].slice(request.offset, request.offset + request.limit), }; }, committedPayload: publicationPayload(), async reload() {}, async close() {}, - }; + }); }, }, }); attachCurrentCoverage(index, auditStore, 'stale-window'); - await assert.rejects( - index.search( + assert.deepStrictEqual( + await index.search( { attribute: 'stale-window', comparator: 'matches', value: 'shoe' }, {}, { @@ -2751,12 +3087,12 @@ describe('FullTextQueryIndex', () => { }, } ), - (error) => error.name === 'DerivedIndexLagError' && error.statusCode === 503 && error.retryable === true + [] ); await index.close(); }); - it('samples companion-gate rejections before classifying window exhaustion', async () => { + it('exhausts companion-gate rejections without returning stale matches', async () => { const auditStore = sharedStore(); const readinessId = 'stale-gate-window'; publishDerivedIndexReadiness(auditStore, readinessId, 'ready'); @@ -2777,8 +3113,8 @@ describe('FullTextQueryIndex', () => { entryForKey: () => ({ version: 2, value: { title: 'shoe' } }), }); attachCurrentCoverage(index, auditStore, readinessId); - await assert.rejects( - index.search( + assert.deepStrictEqual( + await index.search( { attribute: readinessId, comparator: 'matches', value: 'shoe' }, {}, { @@ -2791,13 +3127,13 @@ describe('FullTextQueryIndex', () => { }, } ), - (error) => error.name === 'DerivedIndexLagError' && error.statusCode === 503 && error.retryable === true + [] ); - assert.strictEqual(sourceReads, 1, 'only the exhaustion sample should load a rejected record'); + assert.strictEqual(sourceReads, 0, 'a complete gate can reject all hits without reading records'); await index.close(); }); - it('reports expired companion-gate rejections when they exhaust the search window', async () => { + it('exhausts expired companion-gate rejections beyond the search window', async () => { const auditStore = sharedStore(); const readinessId = 'expired-gate-window'; publishDerivedIndexReadiness(auditStore, readinessId, 'ready'); @@ -2818,8 +3154,8 @@ describe('FullTextQueryIndex', () => { entryForKey: () => ({ version: 1, expiresAt: 0, value: { title: 'shoe' } }), }); attachCurrentCoverage(index, auditStore, readinessId); - await assert.rejects( - index.search( + assert.deepStrictEqual( + await index.search( { attribute: readinessId, comparator: 'matches', value: 'shoe' }, {}, { @@ -2832,9 +3168,9 @@ describe('FullTextQueryIndex', () => { }, } ), - /Expired full-text matches exhausted the 4-result search window/ + [] ); - assert.strictEqual(sourceReads, 4, 'only the bounded exhaustion sample should load rejected records'); + assert.strictEqual(sourceReads, 0, 'a complete gate can reject expired hits without reading records'); await index.close(); }); @@ -2861,7 +3197,7 @@ describe('FullTextQueryIndex', () => { }; }, async openNativeFullTextReader() { - return { + return withTraversal({ async search() { throw Object.assign(new Error('corrupt'), { code: 'E_INDEX_CORRUPT' }); }, @@ -2870,7 +3206,7 @@ describe('FullTextQueryIndex', () => { async close() { if (closeAttempts++ === 0) throw new Error('close failed'); }, - }; + }); }, }, }); @@ -2915,14 +3251,14 @@ describe('FullTextQueryIndex', () => { }; }, async openNativeFullTextReader() { - return { + return withTraversal({ async search() { throw Object.assign(new Error('native storage cannot open'), { code }); }, committedPayload: publicationPayload(), async reload() {}, async close() {}, - }; + }); }, }, }); @@ -2966,7 +3302,7 @@ describe('FullTextQueryIndex', () => { }; }, async openNativeFullTextReader() { - return { + return withTraversal({ async search() { return { total: 1, @@ -2977,7 +3313,7 @@ describe('FullTextQueryIndex', () => { committedPayload: publicationPayload(), async reload() {}, async close() {}, - }; + }); }, }, }); @@ -3029,10 +3365,10 @@ describe('FullTextQueryIndex', () => { async openNativeFullTextReader() { const readerIndex = opens++; let readerHits = publishedHits; - return { + return withTraversal({ async search(request) { pages.push({ readerIndex, offset: request.offset }); - if (readerIndex === 0 && request.offset > 0) { + if (readerIndex === 0 && pages.filter((page) => page.readerIndex === 0).length === 2) { firstStarted.resolve(); await firstSearch.promise; } @@ -3050,7 +3386,7 @@ describe('FullTextQueryIndex', () => { async close() { closes++; }, - }; + }); }, }, }); @@ -3077,7 +3413,7 @@ describe('FullTextQueryIndex', () => { assert.strictEqual(reloads, 0); assert.deepStrictEqual(pages, [ { readerIndex: 0, offset: 0 }, - { readerIndex: 0, offset: 256 }, + { readerIndex: 0, offset: 0 }, { readerIndex: 1, offset: 0 }, ]); await waitFor(() => closes === 1); @@ -3118,7 +3454,7 @@ describe('FullTextQueryIndex', () => { }, async openNativeFullTextReader() { opens++; - return { + return withTraversal({ async search() { return { total: 1, @@ -3131,7 +3467,7 @@ describe('FullTextQueryIndex', () => { reloads++; }, async close() {}, - }; + }); }, }, }); @@ -3189,7 +3525,7 @@ describe('FullTextQueryIndex', () => { throw new Error('transient replacement open failure'); } const first = opens === 1; - return { + return withTraversal({ async search() { if (first) { firstSearchStarted.resolve(); @@ -3206,7 +3542,7 @@ describe('FullTextQueryIndex', () => { async close() { closes++; }, - }; + }); }, }, }); @@ -3229,87 +3565,101 @@ describe('FullTextQueryIndex', () => { assert.strictEqual(closes, opens - 1); }); - it('bounds close while a native search keeps a reader lease', async () => { - const auditStore = sharedStore(); - const readinessId = 'query-reader-close-timeout'; - const storePath = '/unused'; - const storeName = 'query-reader-close-timeout'; - const nativePath = nativeFullTextIndexPath(storePath, storeName); - publishDerivedIndexReadiness(auditStore, readinessId, 'ready'); - const searchStarted = Promise.withResolvers(); - const finishSearch = Promise.withResolvers(); - let closes = 0; - const index = new FullTextQueryIndex({ - Table: { - tableId: 1, - primaryStore: { getEntry: () => ({ version: 1, value: { title: 'shoe' } }) }, - _readTxnForContext: () => undefined, - }, - definition: definition(), - auditStore, - readinessId, - indexId: readinessId, - storePath, - storeName, - sourceGeneration: 'generation', - limits: {}, - binding: { - async runtimeInfo() { - return { - queryClassIsolationMinimumSearchThreads: 1, - limits: queryLimits({ maxSearchWindow: 10, maxSearchBudgetMilliseconds: 100, maxTraceRecords: 10 }), - }; - }, - async openNativeFullTextReader() { - return { - async search() { - searchStarted.resolve(); - await finishSearch.promise; + for (const exceedsBudget of [false, true]) { + it( + exceedsBudget + ? 'bounds close while a native search keeps a reader lease' + : 'lets an in-budget search finish through reader retirement and pause', + async () => { + const auditStore = sharedStore(); + const readinessId = 'query-reader-close-timeout'; + const storePath = '/unused'; + const storeName = 'query-reader-close-timeout'; + const nativePath = nativeFullTextIndexPath(storePath, storeName); + publishDerivedIndexReadiness(auditStore, readinessId, 'ready'); + const searchStarted = Promise.withResolvers(); + const finishSearch = Promise.withResolvers(); + let closes = 0; + const index = new FullTextQueryIndex({ + Table: { + tableId: 1, + primaryStore: { getEntry: () => ({ version: 1, value: { title: 'shoe' } }) }, + _readTxnForContext: () => undefined, + }, + definition: definition(), + auditStore, + readinessId, + indexId: readinessId, + storePath, + storeName, + sourceGeneration: 'generation', + limits: {}, + binding: { + async runtimeInfo() { return { - total: 1, - totalRelation: 'exact', - hits: [{ id: nativeId(1, 'one'), version: '1', score: 1 }], + queryClassIsolationMinimumSearchThreads: 1, + limits: queryLimits({ + maxSearchWindow: 10, + maxSearchBudgetMilliseconds: exceedsBudget ? 100 : 5000, + maxTraceRecords: 10, + }), }; }, - committedPayload: publicationPayload(), - async reload() {}, - async close() { - closes++; + async openNativeFullTextReader() { + return withTraversal({ + async search() { + searchStarted.resolve(); + await finishSearch.promise; + return { + total: 1, + totalRelation: 'exact', + hits: [{ id: nativeId(1, 'one'), version: '1', score: 1 }], + }; + }, + committedPayload: publicationPayload(), + async reload() {}, + async close() { + closes++; + }, + }); }, - }; - }, - }, - }); - attachCurrentCoverage(index, auditStore, readinessId); - const search = index.search( - { attribute: readinessId, comparator: 'matches', value: 'shoe' }, - {}, - { minResults: 1 } + }, + }); + attachCurrentCoverage(index, auditStore, readinessId); + const search = index.search( + { attribute: readinessId, comparator: 'matches', value: 'shoe' }, + {}, + { minResults: 1 } + ); + await searchStarted.promise; + let closeError; + const close = index.close().catch((error) => (closeError = error)); + if (exceedsBudget) { + await waitFor(() => closeError, 3000); + assert.match(closeError.message, /reader drain did not settle/); + assert.strictEqual(closeError.statusCode, 503); + } + const replacement = readyQueryIndex(auditStore, 'replacement-readiness', storeName); + let pauseSettled = false; + const pause = pauseNativeFullTextQueryReaders(nativePath, 'replacement-readiness', 2n).then(() => { + pauseSettled = true; + }); + await new Promise((resolve) => setImmediate(resolve)); + assert.strictEqual(pauseSettled, false); + finishSearch.resolve(); + if (exceedsBudget) await assert.rejects(search, /Full-text search exceeded its execution budget/); + else assert.strictEqual((await search).length, 1); + await close; + if (!exceedsBudget) assert.strictEqual(closeError, undefined); + await pause; + assert.strictEqual(pauseSettled, true); + resumeNativeFullTextQueryReaders(nativePath, 'replacement-readiness', 2n); + await waitFor(() => closes === 1); + await index.close(); + await replacement.close(); + } ); - await searchStarted.promise; - let closeError; - const close = index.close().catch((error) => (closeError = error)); - await new Promise((resolve) => setTimeout(resolve, 1_500)); - assert.match(closeError?.message, /reader drain did not settle/); - assert.strictEqual(closeError.statusCode, 503); - await close; - const replacement = readyQueryIndex(auditStore, 'replacement-readiness', storeName); - let pauseSettled = false; - const pause = pauseNativeFullTextQueryReaders(nativePath, 'replacement-readiness', 2n).then(() => { - pauseSettled = true; - }); - await new Promise((resolve) => setTimeout(resolve, 20)); - assert.strictEqual(pauseSettled, false); - finishSearch.resolve(); - assert.strictEqual((await search).length, 1); - await pause; - assert.strictEqual(pauseSettled, true); - resumeNativeFullTextQueryReaders(nativePath, 'replacement-readiness', 2n); - await waitFor(() => closes === 1); - await index.close(); - await replacement.close(); - }); - + } it('does not let a native reader close failure reject lifecycle shutdown', async () => { const auditStore = sharedStore(); const readinessId = 'query-reader-close-failure'; @@ -3339,7 +3689,7 @@ describe('FullTextQueryIndex', () => { }; }, async openNativeFullTextReader() { - return { + return withTraversal({ async search() { return { total: 1, @@ -3353,7 +3703,7 @@ describe('FullTextQueryIndex', () => { closes++; throw new Error('close failed'); }, - }; + }); }, }, }); @@ -3409,7 +3759,7 @@ describe('FullTextQueryIndex', () => { }; }, async openNativeFullTextReader() { - return { + return withTraversal({ async search() { return { total: 1, @@ -3439,7 +3789,7 @@ describe('FullTextQueryIndex', () => { committedPayload: publicationPayload(), async reload() {}, async close() {}, - }; + }); }, }, }); @@ -3572,7 +3922,7 @@ describe('FullTextQueryIndex', () => { }; }, async openNativeFullTextReader() { - return { + return withTraversal({ async search() { return { total: 2, @@ -3590,7 +3940,7 @@ describe('FullTextQueryIndex', () => { committedPayload: publicationPayload(), async reload() {}, async close() {}, - }; + }); }, }, }); @@ -3647,7 +3997,7 @@ describe('FullTextQueryIndex', () => { }, async openNativeFullTextReader() { opens++; - return { + return withTraversal({ async search() { return { total: 1, @@ -3658,7 +4008,7 @@ describe('FullTextQueryIndex', () => { committedPayload: publicationPayload(), async reload() {}, async close() {}, - }; + }); }, }, }); @@ -3712,7 +4062,7 @@ describe('FullTextQueryIndex', () => { openStarted.resolve(); await finishOpen.promise; } - return { + return withTraversal({ async search() { return { total: 1, @@ -3723,7 +4073,7 @@ describe('FullTextQueryIndex', () => { committedPayload: publicationPayload(), async reload() {}, async close() {}, - }; + }); }, }, }); @@ -3778,7 +4128,7 @@ describe('FullTextQueryIndex', () => { }; }, async openNativeFullTextReader() { - return { + return withTraversal({ async search() { if (oldGeneration) { searchStarted.resolve(); @@ -3795,7 +4145,7 @@ describe('FullTextQueryIndex', () => { async close() { if (oldGeneration) oldCloses++; }, - }; + }); }, }, }); @@ -3907,7 +4257,7 @@ describe('FullTextQueryIndex', () => { }; }, async openNativeFullTextReader() { - return { + return withTraversal({ async search() { return { total: 1, @@ -3918,7 +4268,7 @@ describe('FullTextQueryIndex', () => { committedPayload: publicationPayload(), async reload() {}, async close() {}, - }; + }); }, }, }); @@ -3984,7 +4334,7 @@ describe('FullTextQueryIndex', () => { }; }, async openNativeFullTextReader() { - return { + return withTraversal({ async search() { searchStarted.resolve(); await finishSearch.promise; @@ -3997,7 +4347,7 @@ describe('FullTextQueryIndex', () => { committedPayload: publicationPayload(), async reload() {}, async close() {}, - }; + }); }, }, }); @@ -4118,7 +4468,7 @@ describe('FullTextQueryIndex', () => { }; }, async openNativeFullTextReader() { - return { + return withTraversal({ async search() { return { total: 1, @@ -4129,7 +4479,7 @@ describe('FullTextQueryIndex', () => { committedPayload: publicationPayload(), async reload() {}, async close() {}, - }; + }); }, }, }); @@ -4296,7 +4646,7 @@ describe('FullTextQueryIndex', () => { }; }, async openNativeFullTextReader() { - return { + return withTraversal({ async search() { return { total: 1, @@ -4307,7 +4657,7 @@ describe('FullTextQueryIndex', () => { committedPayload: encodeFullTextCursorPayload({ format: 1, logs: { local: 1 } }), async reload() {}, async close() {}, - }; + }); }, }, }); diff --git a/unitTests/resources/fullTextRuntimeConfiguration.test.js b/unitTests/resources/fullTextRuntimeConfiguration.test.js new file mode 100644 index 0000000000..1d2a5572b5 --- /dev/null +++ b/unitTests/resources/fullTextRuntimeConfiguration.test.js @@ -0,0 +1,76 @@ +const { getFullTextTestConfiguration, setFullTextTestConfiguration } = require('./fullTextTestConfiguration'); +require('../testUtils'); +const assert = require('node:assert'); +const { once } = require('node:events'); + +const { startWorker, onMessageByType } = require('#src/server/threads/manageThreads'); +const { supportsPublishedFullTextBinding } = require('./fullTextNativePlatform'); +onMessageByType('full-text-runtime-test-response', () => {}); +function response(worker, action, timeout = 60000) { + return new Promise((resolve, reject) => { + const timer = setTimeout( + () => finish(new Error(`Full-text runtime worker did not answer: ${action ?? 'ready'}`)), + timeout + ); + const receive = (event) => { + if (event.type === 'full-text-runtime-test-response') finish(undefined, event.message); + }; + function finish(error, value) { + clearTimeout(timer); + worker.off('message', receive); + if (error) reject(error); + else resolve(value); + } + worker.on('message', receive); + if (action) + worker.postMessage({ + type: 'full-text-runtime-test-command', + message: { originator: 0, ...(typeof action === 'string' ? { operation: action } : action) }, + }); + }); +} + +async function terminateWorker(worker) { + if (!worker || worker.threadId < 0) return; + const exited = once(worker, 'exit', { signal: AbortSignal.timeout(10000) }); + await Promise.all([exited, worker.terminate()]); +} + +describe('full-text native runtime configuration', () => { + it('keeps one native runtime configuration across replacement workers and their application workers', async function () { + this.timeout(300000); + if (!(await supportsPublishedFullTextBinding())) this.skip(); + const { getFullTextRuntimeConfiguration } = require('#src/resources/indexes/fullTextRuntimeConfiguration'); + const { resolveFullTextSearchLimits } = require('#src/resources/indexes/fullTextSearchLimits'); + const runtime = resolveFullTextSearchLimits(getFullTextRuntimeConfiguration()).runtime; + const previous = getFullTextTestConfiguration(); + const workers = []; + try { + for (let n = 0; n < 4; n++) { + if (n === 1) + setFullTextTestConfiguration({ + ...previous, + runtime: { maxSearchThreads: runtime.maxSearchThreads + 1 }, + }); + if (n === 2) setFullTextTestConfiguration({ ...previous, requestBudgetMilliseconds: 0 }); + if (n === 3) setFullTextTestConfiguration({ ...previous, runtime: { maxSearchThreads: 0 } }); + const worker = startWorker(require.resolve('./fixtures/fullTextRuntimeConfigurationWorker.js'), { + name: `full-text-runtime-${n}`, + autoRestart: false, + workerIndex: n, + threadCount: 4, + }); + workers.push(worker); + assert.strictEqual((await response(worker)).ready, true); + const result = await response(worker, 'runtime-configuration'); + assert.deepStrictEqual(result.value?.runtime, runtime, JSON.stringify(result)); + assert.strictEqual(result.value.changedCode, 'E_RESOURCE_LIMIT'); + assert.strictEqual(result.value.rawChildReady, true); + } + assert.deepStrictEqual((await response(workers[0], 'runtime-configuration')).value.runtime, runtime); + } finally { + setFullTextTestConfiguration(previous); + for (const worker of workers) await terminateWorker(worker); + } + }); +}); diff --git a/unitTests/resources/fullTextSearchLimits.test.js b/unitTests/resources/fullTextSearchLimits.test.js new file mode 100644 index 0000000000..42ee340da1 --- /dev/null +++ b/unitTests/resources/fullTextSearchLimits.test.js @@ -0,0 +1,157 @@ +const assert = require('node:assert'); +const { + DEFAULT_FULL_TEXT_SEARCH_LIMITS, + resolveFullTextSearchLimits, +} = require('#src/resources/indexes/fullTextSearchLimits'); + +describe('full-text search resource limits', () => { + it('adopts new runtime budgets only in a fresh process and retains invalid snapshots', async function () { + const { supportsPublishedFullTextBinding } = require('./fullTextNativePlatform'); + if (!(await supportsPublishedFullTextBinding())) this.skip(); + const { execFileSync } = require('node:child_process'); + const fixture = require.resolve('./fixtures/fullTextRuntimeConfigurationProcess.cjs'); + const unmanaged = execFileSync(process.execPath, [fixture, '256', 'unmanaged'], { + encoding: 'utf8', + timeout: 15000, + }); + assert.strictEqual(JSON.parse(unmanaged.trim().split('\n').at(-1)).unmanaged, 'ready'); + const early = execFileSync(process.execPath, [fixture, '256', 'early-worker'], { + encoding: 'utf8', + timeout: 15000, + }); + const rejection = JSON.parse(early.trim().split('\n').at(-1)).unmanaged; + assert.strictEqual(rejection.status, 500); + assert.strictEqual(rejection.name, 'FullTextWorkerInitializationError'); + assert.strictEqual(rejection.retryable, false); + assert.strictEqual(rejection.optionalRuntimeAvailable, false); + assert.match(rejection.error, /Initialize Harper before creating application workers/); + for (const value of [257, 258]) { + const result = execFileSync(process.execPath, [fixture, String(value)], { encoding: 'utf8', timeout: 15000 }); + assert.strictEqual(JSON.parse(result.trim().split('\n').at(-1)).runtime.maxSearchThreads, value); + } + assert.throws( + () => execFileSync(process.execPath, [fixture, '0'], { encoding: 'utf8', timeout: 15000 }), + (error) => /positive safe integer/.test(error.stdout) + ); + }); + + it('resolves missing settings without changing the defaults', () => { + assert.deepStrictEqual(resolveFullTextSearchLimits(undefined), DEFAULT_FULL_TEXT_SEARCH_LIMITS); + assert(Object.isFrozen(DEFAULT_FULL_TEXT_SEARCH_LIMITS.runtime)); + }); + + it('observes changed work limits when a live configuration object is reused', () => { + const config = { maxRawHitsPerRequest: 10, requestBudgetMilliseconds: 100 }; + assert.strictEqual(resolveFullTextSearchLimits(config).maxRawHitsPerRequest, 10); + config.maxRawHitsPerRequest = 20; + config.requestBudgetMilliseconds = 200; + const updated = resolveFullTextSearchLimits(config); + assert.strictEqual(updated.maxRawHitsPerRequest, 20); + assert.strictEqual(updated.requestBudgetMilliseconds, 200); + }); + + it('accepts integer strings and independently merges the native process budget', () => { + const limits = resolveFullTextSearchLimits({ + requestBudgetMilliseconds: '7', + runtime: { maxResidentIndexes: '256' }, + }); + assert.strictEqual(limits.requestBudgetMilliseconds, 7); + assert.strictEqual(limits.maxRawHitsPerRequest, DEFAULT_FULL_TEXT_SEARCH_LIMITS.maxRawHitsPerRequest); + assert.strictEqual(limits.runtime.maxResidentIndexes, 256); + assert.strictEqual(limits.runtime.maxSearchThreads, DEFAULT_FULL_TEXT_SEARCH_LIMITS.runtime.maxSearchThreads); + }); + + it('rejects invalid full-text settings through the actual root configuration validator', () => { + const { configValidator } = require('#src/validation/configValidator'); + const errors = (fullText) => + ( + configValidator( + { rootPath: require('node:path').resolve(__dirname, '../..'), storage: { writeAsync: false, fullText } }, + true + ).error?.details ?? [] + ).filter(({ path }) => path[0] === 'storage' && path[1] === 'fullText'); + assert.deepStrictEqual(errors({ requestBudgetMilliseconds: 7, runtime: { maxResidentIndexes: 256 } }), []); + for (const value of [ + { requestBudgetMilliseconds: 0 }, + { maxCursor: 4 }, + { runtime: { maxSearchThreads: null } }, + { requestBudgetMilliseconds: -1 }, + ]) + assert(errors(value).length > 0, JSON.stringify(value)); + }); + + it('reports a non-Error resolver failure through root configuration validation', () => { + const { configValidator } = require('#src/validation/configValidator'); + const result = configValidator( + { + rootPath: require('node:path').resolve(__dirname, '../..'), + storage: { + writeAsync: false, + fullText: { + get runtime() { + throw null; + }, + }, + }, + }, + true + ); + const detail = result.error?.details.find(({ path }) => path[0] === 'storage' && path[1] === 'fullText'); + assert(detail); + assert.strictEqual(detail.message, 'null'); + }); + + it('keeps the shipped YAML defaults and schema defaults consistent with the resolver', () => { + const { readFileSync } = require('node:fs'); + const { resolve } = require('node:path'); + const { parse } = require('yaml'); + const root = resolve(__dirname, '../..'); + const yaml = parse(readFileSync(resolve(root, 'static/defaultConfig.yaml'), 'utf8')); + assert.deepStrictEqual(yaml.storage.fullText, DEFAULT_FULL_TEXT_SEARCH_LIMITS); + const schema = JSON.parse(readFileSync(resolve(root, 'config-root.schema.json'), 'utf8')); + const policy = schema.properties.storage.properties.fullText; + assert.strictEqual(policy.additionalProperties, false); + assert.strictEqual(policy.properties.runtime.additionalProperties, false); + for (const [name, value] of Object.entries(DEFAULT_FULL_TEXT_SEARCH_LIMITS)) { + if (name === 'runtime') { + for (const [key, limit] of Object.entries(value)) + assert.strictEqual(policy.properties.runtime.properties[key].default, limit); + } else assert.strictEqual(policy.properties[name].default, value); + } + }); + + it('exposes every limit through the conventional configuration parameter map', () => { + const { CONFIG_PARAM_MAP, CONFIG_PARAMS } = require('#src/utility/hdbTerms'); + const { flattenConfig } = require('#src/config/configUtils'); + const flattened = flattenConfig({ storage: { fullText: structuredClone(DEFAULT_FULL_TEXT_SEARCH_LIMITS) } }); + assert.deepStrictEqual(flattened.storage_fulltext_runtime, DEFAULT_FULL_TEXT_SEARCH_LIMITS.runtime); + assert.strictEqual(CONFIG_PARAM_MAP.storage_fulltext_runtime, CONFIG_PARAMS.STORAGE_FULL_TEXT_RUNTIME); + for (const [name, value] of Object.entries(DEFAULT_FULL_TEXT_SEARCH_LIMITS)) { + const check = (path, expected) => { + const parameter = `storage_fullText_${path}`; + assert.strictEqual(CONFIG_PARAM_MAP[parameter.toLowerCase()], parameter); + assert.strictEqual(flattened[parameter.toLowerCase()], expected); + }; + if (name === 'runtime') for (const [key, limit] of Object.entries(value)) check(`runtime_${key}`, limit); + else check(name, value); + } + }); + + it('rejects missing-shape, unknown and unbounded policy values', () => { + for (const value of [null, [], true, 'settings', new Date(), Object.create({ requestBudgetMilliseconds: 7 })]) + assert.throws(() => resolveFullTextSearchLimits(value)); + for (const value of [null, 0, -1, 1.5, NaN, Infinity, Number.MAX_SAFE_INTEGER + 1, '', '1.5', '1e3', true]) { + assert.throws(() => resolveFullTextSearchLimits({ requestBudgetMilliseconds: value }), /positive safe integer/); + assert.throws(() => resolveFullTextSearchLimits({ maxRawHitsPerRequest: value }), /positive safe integer/); + assert.throws(() => resolveFullTextSearchLimits({ maxResultsPerRequest: value }), /positive safe integer/); + assert.throws( + () => resolveFullTextSearchLimits({ runtime: { maxSearchThreads: value } }), + /positive safe integer/ + ); + } + assert.throws(() => resolveFullTextSearchLimits({ maxCursor: 4 }), /Unknown/); + assert.throws(() => resolveFullTextSearchLimits({ runtime: { threads: 4 } }), /Unknown/); + assert.throws(() => resolveFullTextSearchLimits({ constructor: 4 }), /Unknown/); + assert.throws(() => resolveFullTextSearchLimits({ runtime: { maxResidentIndexes: 0x100000000 } }), /u32/); + }); +}); diff --git a/unitTests/resources/fullTextTestConfiguration.js b/unitTests/resources/fullTextTestConfiguration.js new file mode 100644 index 0000000000..84292498b1 --- /dev/null +++ b/unitTests/resources/fullTextTestConfiguration.js @@ -0,0 +1,13 @@ +const env = require('#src/utility/environment/environmentManager'); +const { CONFIG_PARAMS } = require('#src/utility/hdbTerms'); +const parameters = { + requestBudgetMilliseconds: CONFIG_PARAMS.STORAGE_FULL_TEXT_REQUEST_BUDGET_MILLISECONDS, + maxRawHitsPerRequest: CONFIG_PARAMS.STORAGE_FULL_TEXT_MAX_RAW_HITS_PER_REQUEST, + maxResultsPerRequest: CONFIG_PARAMS.STORAGE_FULL_TEXT_MAX_RESULTS_PER_REQUEST, + runtime: CONFIG_PARAMS.STORAGE_FULL_TEXT_RUNTIME, +}; +exports.getFullTextTestConfiguration = () => + Object.fromEntries(Object.entries(parameters).map(([name, parameter]) => [name, env.get(parameter)])); +exports.setFullTextTestConfiguration = (value) => { + for (const [name, parameter] of Object.entries(parameters)) env.setProperty(parameter, value?.[name]); +}; diff --git a/unitTests/resources/nativeFullTextDerivedIndexLifecycle.test.js b/unitTests/resources/nativeFullTextDerivedIndexLifecycle.test.js index 8817b68d1c..b90f13ac0d 100644 --- a/unitTests/resources/nativeFullTextDerivedIndexLifecycle.test.js +++ b/unitTests/resources/nativeFullTextDerivedIndexLifecycle.test.js @@ -13,6 +13,7 @@ const { loadFullTextNativeBinding } = require('#src/resources/indexes/fullTextNa const { DERIVED_INDEX_ACCEPTED, DERIVED_INDEX_DEFERRED } = require('#src/resources/derivedIndexRuntime'); const { waitFor } = require('../waitFor'); const fullTextPackageVersion = require('../../package.json').optionalDependencies['@harperfast/fulltext']; +const { supportsPublishedFullTextBinding } = require('./fullTextNativePlatform'); const limits = { indexingThreads: 2, @@ -46,6 +47,7 @@ class FakeNativeModule { applyMutationBatch() {} publish() {} close() {} + searchPage() {} }; this.runtimeInfoCalls = 0; this.inspection = { state: 'missing' }; @@ -62,11 +64,11 @@ class FakeNativeModule { return { packageVersion: 'test', tantivyVersion: 'test', - nativeAbiVersion: 5, + nativeAbiVersion: 9, queryClassIsolationMinimumSearchThreads: 2, lifecycleApiVersion: 1, mutationBatchApiVersion: 5, - queryApiVersion: 3, + queryApiVersion: 4, storageBackends: ['native'], limits: { maxCommitPayloadBytes: this.maxCommitPayloadBytes, @@ -75,6 +77,7 @@ class FakeNativeModule { maxCandidateIds: 1_024, maxCandidateBytes: 1024 * 1024, maxSearchWindow: 10_000, + maxSearchPageSize: 10_000, maxAutocompleteResults: 100, maxSearchResponseBytes: 8 * 1024 * 1024, maxSearchBudgetMilliseconds: 30_000, @@ -140,22 +143,16 @@ describe('NativeFullTextDerivedIndexLifecycle', () => { }); it('loads the published registry package and persists a searchable checkpoint', async function () { - const glibcVersion = - process.platform === 'linux' ? process.report?.getReport().header.glibcVersionRuntime : undefined; - const supportedTarget = - (process.platform === 'darwin' && process.arch === 'arm64') || - (process.platform === 'linux' && glibcVersion && ['arm64', 'x64'].includes(process.arch)) || - (process.platform === 'win32' && process.arch === 'x64'); - if (!supportedTarget) this.skip(); + if (!(await supportsPublishedFullTextBinding())) this.skip(); const binding = await loadFullTextNativeBinding(); const runtime = await binding.runtimeInfo(); assert.strictEqual(runtime.packageVersion, fullTextPackageVersion); assert.strictEqual(runtime.tantivyVersion, '0.26.2'); - assert.strictEqual(runtime.nativeAbiVersion, 8); + assert.strictEqual(runtime.nativeAbiVersion, 9); assert.strictEqual(runtime.lifecycleApiVersion, 1); assert.strictEqual(runtime.mutationBatchApiVersion, 5); - assert.strictEqual(runtime.queryApiVersion, 3); + assert.strictEqual(runtime.queryApiVersion, 4); assert.strictEqual(runtime.queryClassIsolationMinimumSearchThreads, 2); const lifecycle = new NativeFullTextDerivedIndexLifecycle(options(storePath, binding, { surfaceTerms: true })); await lifecycle.initialize(); @@ -482,11 +479,11 @@ describe('NativeFullTextDerivedIndexLifecycle', () => { binding.runtimeInfo = async () => ({ packageVersion: 'test', tantivyVersion: 'test', - nativeAbiVersion: 5, + nativeAbiVersion: 9, queryClassIsolationMinimumSearchThreads: 2, lifecycleApiVersion: 2, mutationBatchApiVersion: 5, - queryApiVersion: 3, + queryApiVersion: 4, storageBackends: ['native'], limits: { maxCommitPayloadBytes: 64 * 1024, @@ -495,6 +492,7 @@ describe('NativeFullTextDerivedIndexLifecycle', () => { maxCandidateIds: 1_024, maxCandidateBytes: 1024 * 1024, maxSearchWindow: 10_000, + maxSearchPageSize: 10_000, maxAutocompleteResults: 100, maxSearchResponseBytes: 8 * 1024 * 1024, maxSearchBudgetMilliseconds: 30_000, @@ -511,7 +509,7 @@ describe('NativeFullTextDerivedIndexLifecycle', () => { binding.runtimeInfo = async () => ({ packageVersion: 'test', tantivyVersion: 'test', - nativeAbiVersion: 5, + nativeAbiVersion: 9, queryClassIsolationMinimumSearchThreads: 2, lifecycleApiVersion: 1, storageBackends: ['native'], @@ -520,6 +518,7 @@ describe('NativeFullTextDerivedIndexLifecycle', () => { maxRecordIdBytes: 4_096, maxRecordVersionBytes: 4_096, maxSearchWindow: 10_000, + maxSearchPageSize: 10_000, maxAutocompleteResults: 100, maxSearchResponseBytes: 8 * 1024 * 1024, maxSearchBudgetMilliseconds: 30_000, diff --git a/unitTests/resources/nativeFullTextTableSearch.test.js b/unitTests/resources/nativeFullTextTableSearch.test.js index 63d37480f7..01d5c9c0f4 100644 --- a/unitTests/resources/nativeFullTextTableSearch.test.js +++ b/unitTests/resources/nativeFullTextTableSearch.test.js @@ -1,5 +1,7 @@ 'use strict'; +const { getFullTextTestConfiguration, setFullTextTestConfiguration } = require('./fullTextTestConfiguration'); + require('../testUtils'); const assert = require('node:assert'); const { setupTestDBPath } = require('../testUtils'); @@ -10,15 +12,7 @@ const { parseQuery } = require('#src/resources/search'); const { setMainIsWorker } = require('#src/server/threads/manageThreads'); const { waitFor } = require('../waitFor'); -function supportsPublishedBinding() { - const glibcVersion = - process.platform === 'linux' ? process.report?.getReport().header.glibcVersionRuntime : undefined; - return ( - (process.platform === 'darwin' && process.arch === 'arm64') || - (process.platform === 'linux' && glibcVersion && ['arm64', 'x64'].includes(process.arch)) || - (process.platform === 'win32' && process.arch === 'x64') - ); -} +const { supportsPublishedFullTextBinding } = require('./fullTextNativePlatform'); async function collect(iterable) { const values = []; @@ -45,8 +39,8 @@ function definition(name, fields) { describe('published native full-text Table.search integration', () => { let Product; - before(function () { - if (process.env.HARPER_STORAGE_ENGINE === 'lmdb' || !supportsPublishedBinding()) this.skip(); + before(async function () { + if (process.env.HARPER_STORAGE_ENGINE === 'lmdb' || !(await supportsPublishedFullTextBinding())) this.skip(); setupTestDBPath(); setMainIsWorker(true); }); @@ -68,6 +62,46 @@ describe('published native full-text Table.search integration', () => { setFullTextNativeBindingForTests(undefined); }); + it('keeps custom iterable searches and re-evaluates later offset requests', async () => { + Product = table({ + database: `fulltext-offset-contract-${Date.now()}`, + table: 'Product', + audit: true, + attributes: [ + { name: 'id', type: 'ID', isPrimaryKey: true }, + { name: 'title', type: 'String' }, + ], + fullTextIndexes: [definition('catalogSearch', [{ name: 'title', weight: 1 }])], + }); + await Product.put(['a', 'b', 'c'].map((id) => ({ id, title: 'widget' }))); + await waitFor(() => fullTextDerivedIndexReadiness(Product, 'catalogSearch').state === 'ready', 30000); + const query = { + conditions: [ + { + attribute: 'catalogSearch', + comparator: 'matches', + value: 'widget', + maxIndexLagMilliseconds: 0, + waitForIndexMilliseconds: 5000, + }, + ], + limit: 1, + }; + class Filtered extends Product { + search(target) { + return super.search({ ...target, rowFilter: (record) => record.id !== 'a' }); + } + } + assert.strictEqual(Product.searchPage, undefined); + assert.strictEqual(new Product().searchPage, undefined); + assert.strictEqual(typeof Filtered.search(query)[Symbol.asyncIterator], 'function'); + assert.deepStrictEqual(ids(await collect(Filtered.search(query))), ['b']); + assert.deepStrictEqual(ids(await collect(Filtered.search({ ...query, offset: 1 }))), ['c']); + await Product.delete('b'); + assert.deepStrictEqual(ids(await collect(Filtered.search(query))), ['c']); + assert.deepStrictEqual(await collect(Filtered.search({ ...query, offset: 1 })), []); + }); + it('executes every query mode, negation, REST syntax, field selection, and same-index OR', async () => { const searchDefinition = definition('catalogSearch', [ { name: 'title', weight: 2, highlight: true }, @@ -90,7 +124,7 @@ describe('published native full-text Table.search integration', () => { await Product.put('one', { title: 'Waterproof Trail Running Shoes', tags: ['outdoor', 'trail'] }); await Product.put('two', { title: 'Waterproof Road Shoes', tags: ['outdoor', 'road'] }); await Product.put('three', { title: 'Wireless Headphones', tags: ['electronics'] }); - await waitFor(() => fullTextDerivedIndexReadiness(Product, 'catalogSearch').state === 'ready', 30_000); + await waitFor(() => fullTextDerivedIndexReadiness(Product, 'catalogSearch').state === 'ready', 60_000); const search = (comparator, value, options = {}) => collect( @@ -278,7 +312,7 @@ describe('published native full-text Table.search integration', () => { category: index < 50 ? 'other' : 'target', })) ); - await waitFor(() => fullTextDerivedIndexReadiness(Product, 'catalogSearch').state === 'ready', 30_000); + await waitFor(() => fullTextDerivedIndexReadiness(Product, 'catalogSearch').state === 'ready', 60_000); const originalGetEntry = Product.primaryStore.getEntry; let sourceReads = 0; @@ -327,7 +361,7 @@ describe('published native full-text Table.search integration', () => { await Product.put( insertionOrder.slice(start, start + 100).map((id) => ({ id, title: 'identical catalog product' })) ); - await waitFor(() => fullTextDerivedIndexReadiness(Product, 'catalogSearch').state === 'ready', 30_000); + await waitFor(() => fullTextDerivedIndexReadiness(Product, 'catalogSearch').state === 'ready', 60_000); const search = (limit) => collect( @@ -357,4 +391,180 @@ describe('published native full-text Table.search integration', () => { expected.slice(0, -5) ); }); + it('keeps native ordinary and traversal prefixes consistent across score, ID and page-size boundaries', async () => { + const { mkdtemp, rm } = require('node:fs/promises'); + const { join } = require('node:path'); + const binding = await loadFullTextNativeBinding(); + const info = await binding.runtimeInfo(); + const path = await mkdtemp(join(__dirname, '../envDir/', String(process.pid), 'fulltext-prefix-')); + const options = { + path, + indexId: 'prefix-test', + generation: '1', + fields: [{ name: 'title' }], + analyzer: 'english@2', + surfaceTerms: true, + limits: { + indexingThreads: 1, + searchThreads: Math.max(2, info.queryClassIsolationMinimumSearchThreads), + writerMemoryBytes: 30000000, + maxQueuedCommands: 16, + maxQueuedBytes: 8388608, + maxBatchBytes: 8388608, + }, + }; + let writer, reader; + try { + writer = await binding.openNativeFullTextIndex(options); + for (let start = 0; start < 1024; start += 128) { + await writer.applyMutationBatch({ + upserts: Array.from({ length: 128 }, (_, i) => ({ + id: `${start + i}-é-😀`, + version: String((start + i) % 3), + fields: { title: (start + i) % 3 ? 'widget catalog' : 'widget widget catalog' }, + })), + }); + await writer.commit(); + } + reader = await binding.openNativeFullTextReader(options); + for (const query of [ + { text: 'widget catalog', mode: 'any' }, + { text: 'widg', mode: 'prefix' }, + { text: 'widg', mode: 'fuzzy-prefix' }, + { operator: 'and', clauses: [{ text: 'widget' }, { operator: 'not', clause: { text: 'absent' } }] }, + ]) { + const ordinary = await reader.search({ + query, + limit: query.mode?.includes('prefix') ? info.limits.maxAutocompleteResults : 1024, + exactTotal: true, + }); + assert.strictEqual(ordinary.totalRelation, 'exact'); + assert.deepStrictEqual((await reader.searchPage({ query, limit: ordinary.hits.length })).hits, ordinary.hits); + for (const limit of [17, 73, 113, 257]) { + const page = await reader.searchPage({ query, limit }); + const sharedLength = Math.min(ordinary.hits.length, page.hits.length); + assert.strictEqual(sharedLength, Math.min(ordinary.hits.length, limit)); + assert.deepStrictEqual(page.hits.slice(0, sharedLength), ordinary.hits.slice(0, sharedLength)); + } + const first = await reader.searchPage({ query, limit: 73 }); + const walk = async (limit) => { + const hits = []; + let continuation = first.continuation; + do { + const page = await reader.searchPage({ query, limit, continuation }); + assert.strictEqual(page.total, 1024); + hits.push(...page.hits); + continuation = page.continuation; + } while (continuation); + return hits; + }; + const tail = await walk(17); + assert.deepStrictEqual(tail, await walk(113)); + const ordered = [...first.hits, ...tail]; + for (let n = 1; n < ordered.length; n++) { + if (ordered[n - 1].score === ordered[n].score) + assert( + Buffer.compare(Buffer.from(ordered[n - 1].id), Buffer.from(ordered[n].id)) < 0, + 'equal-score native hits must advance by document-ID bytes' + ); + } + const small = await reader.searchPage({ query, limit: 17, continuation: first.continuation }); + const large = await reader.searchPage({ query, limit: 113, continuation: first.continuation }); + assert.deepStrictEqual(small.hits, large.hits.slice(0, 17)); + } + } finally { + await reader?.close(); + await writer?.close(); + await rm(path, { recursive: true, force: true }); + } + }); + + it('fills filtered ordinary and autocomplete pages beyond the native offset and hit windows', async () => { + const { toBufferKey } = require('ordered-binary'); + const info = await (await loadFullTextNativeBinding()).runtimeInfo(); + const count = info.limits.maxSearchWindow + 25; + Product = table({ + database: `fulltext-native-traversal-${Date.now()}`, + table: 'Product', + audit: true, + attributes: [ + { name: 'id', type: 'ID', isPrimaryKey: true }, + { name: 'title', type: 'String' }, + ], + fullTextIndexes: [definition('catalogSearch', [{ name: 'title', weight: 1 }])], + }); + const keys = Array.from({ length: count }, (_, i) => `product-${String(i).padStart(5, '0')}`); + for (let start = 0; start < count; start += 250) + await Product.put(keys.slice(start, start + 250).map((id) => ({ id, title: 'widget catalog' }))); + await waitFor(() => fullTextDerivedIndexReadiness(Product, 'catalogSearch').state === 'ready', 60_000); + const ordered = keys + .slice() + .sort((a, b) => + Buffer.compare( + Buffer.from(toBufferKey(a).toString('base64url')), + Buffer.from(toBufferKey(b).toString('base64url')) + ) + ); + const search = (comparator, value, options = {}) => + collect( + Product.search({ + conditions: [ + { + attribute: 'catalogSearch', + comparator, + value, + maxIndexLagMilliseconds: 0, + waitForIndexMilliseconds: 30_000, + }, + ], + limit: 25, + ...options, + }) + ); + const tail = new Set(ordered.slice(info.limits.maxSearchWindow)); + + const previousPolicy = getFullTextTestConfiguration(); + try { + setFullTextTestConfiguration({ ...previousPolicy, maxRawHitsPerRequest: 128 }); + await assert.rejects( + search('matches', 'widget catalog', { rowFilter: (record) => tail.has(record.id) }), + (error) => error.statusCode === 400 && error.message.includes('storage.fullText.maxRawHitsPerRequest') + ); + } finally { + setFullTextTestConfiguration(previousPolicy); + } + + await assert.rejects( + search('matches', 'widget catalog', { rowFilter: (record) => tail.has(record.id) }), + (error) => error.statusCode === 400 && /maxRawHitsPerRequest/.test(error.message) + ); + setFullTextTestConfiguration({ + ...previousPolicy, + maxRawHitsPerRequest: count * 3, + maxResultsPerRequest: count, + }); + try { + assert.deepStrictEqual( + (await search('matches', 'widget catalog', { rowFilter: (record) => tail.has(record.id) })).map(({ id }) => id), + ordered.slice(info.limits.maxSearchWindow) + ); + assert.deepStrictEqual( + (await search('matches', 'widget catalog', { offset: info.limits.maxSearchWindow })).map(({ id }) => id), + ordered.slice(info.limits.maxSearchWindow) + ); + const first = new Set(ordered.slice(0, info.limits.maxAutocompleteResults)); + for (const comparator of ['matches_prefix', 'matches_fuzzy_prefix']) { + assert.deepStrictEqual( + (await search(comparator, 'widg', { rowFilter: (record) => !first.has(record.id) })).map(({ id }) => id), + ordered.slice(info.limits.maxAutocompleteResults, info.limits.maxAutocompleteResults + 25) + ); + assert.deepStrictEqual( + (await search(comparator, 'widg', { limit: info.limits.maxAutocompleteResults + 25 })).map(({ id }) => id), + ordered.slice(0, info.limits.maxAutocompleteResults + 25) + ); + } + } finally { + setFullTextTestConfiguration(previousPolicy); + } + }); }); diff --git a/utility/hdbTerms.ts b/utility/hdbTerms.ts index 65370add57..2eab5083a7 100644 --- a/utility/hdbTerms.ts +++ b/utility/hdbTerms.ts @@ -753,6 +753,16 @@ export const CONFIG_PARAMS = { REPLICATION_LOGGING_TAG: 'replication_logging_tag', ROOTPATH: 'rootPath', SERIALIZATION_BIGINT: 'serialization_bigInt', + STORAGE_FULL_TEXT_REQUEST_BUDGET_MILLISECONDS: 'storage_fullText_requestBudgetMilliseconds', + STORAGE_FULL_TEXT_MAX_RAW_HITS_PER_REQUEST: 'storage_fullText_maxRawHitsPerRequest', + STORAGE_FULL_TEXT_MAX_RESULTS_PER_REQUEST: 'storage_fullText_maxResultsPerRequest', + STORAGE_FULL_TEXT_RUNTIME: 'storage_fullText_runtime', + STORAGE_FULL_TEXT_RUNTIME_MAX_RESIDENT_INDEXES: 'storage_fullText_runtime_maxResidentIndexes', + STORAGE_FULL_TEXT_RUNTIME_MAX_INDEXING_THREADS: 'storage_fullText_runtime_maxIndexingThreads', + STORAGE_FULL_TEXT_RUNTIME_MAX_SEARCH_THREADS: 'storage_fullText_runtime_maxSearchThreads', + STORAGE_FULL_TEXT_RUNTIME_MAX_WRITER_MEMORY_BYTES: 'storage_fullText_runtime_maxWriterMemoryBytes', + STORAGE_FULL_TEXT_RUNTIME_MAX_QUEUED_BYTES: 'storage_fullText_runtime_maxQueuedBytes', + STORAGE_FULL_TEXT_RUNTIME_MAX_EXPENSIVE_SEARCHES: 'storage_fullText_runtime_maxExpensiveSearches', STORAGE_WRITEASYNC: 'storage_writeAsync', STORAGE_OVERLAPPINGSYNC: 'storage_overlappingSync', STORAGE_CACHING: 'storage_caching', diff --git a/validation/configValidator.ts b/validation/configValidator.ts index f699bc6cd6..a97ebf6d74 100644 --- a/validation/configValidator.ts +++ b/validation/configValidator.ts @@ -2,6 +2,7 @@ import fs from 'fs-extra'; import Joi from 'joi'; +import { resolveFullTextSearchLimits } from '../resources/indexes/fullTextSearchLimits.ts'; import * as os from 'node:os'; const { boolean, string, number, array } = Joi.types(); const v8Flag = string.trim().allow('').pattern(/^--/, 'V8 flag starting with --'); @@ -553,6 +554,16 @@ export function configValidator(configJson, skipFsValidation = false) { }) ), storage: Joi.object({ + fullText: Joi.object() + .custom((value, helpers) => { + try { + resolveFullTextSearchLimits(value); + return value; + } catch (error) { + return helpers.message({ custom: error instanceof Error ? error.message : String(error) }); + } + }) + .optional(), writeAsync: boolean.required(), overlappingSync: boolean.optional(), caching: boolean.optional(),